From 33f41937f342184497d0f8c08f2e104648373329 Mon Sep 17 00:00:00 2001 From: Brad Zacher Date: Fri, 5 May 2023 15:58:19 +0800 Subject: [PATCH 1/2] feat(rule-tester): add support for snapshot testing rules --- packages/rule-tester/package.json | 5 +- packages/rule-tester/src/RuleTester.ts | 407 ++++++++++-------- packages/rule-tester/src/TestFramework.ts | 65 +++ .../src/snapshot/expectSnapshot.ts | 234 ++++++++++ .../rule-tester/src/snapshot/renderErrors.ts | 86 ++++ packages/rule-tester/src/snapshot/types.ts | 48 +++ .../rule-tester/src/types/RuleTesterConfig.ts | 13 + .../rule-tester/src/utils/config-schema.ts | 19 + yarn.lock | 5 + 9 files changed, 709 insertions(+), 173 deletions(-) create mode 100644 packages/rule-tester/src/snapshot/expectSnapshot.ts create mode 100644 packages/rule-tester/src/snapshot/renderErrors.ts create mode 100644 packages/rule-tester/src/snapshot/types.ts diff --git a/packages/rule-tester/package.json b/packages/rule-tester/package.json index 5b38b4895e4f..5f97e5fb7473 100644 --- a/packages/rule-tester/package.json +++ b/packages/rule-tester/package.json @@ -49,11 +49,14 @@ "dependencies": { "@typescript-eslint/typescript-estree": "5.59.1", "@typescript-eslint/utils": "5.59.1", + "ajv": "^6.10.0", "lodash.merge": "4.6.2", "semver": "^7.3.7", - "ajv": "^6.10.0" + "make-dir": "^3.1.0", + "std-env": "^3.3.3" }, "peerDependencies": { + "@babel/code-frame": "^7.21.4", "@eslint/eslintrc": ">=2", "eslint": ">=8" }, diff --git a/packages/rule-tester/src/RuleTester.ts b/packages/rule-tester/src/RuleTester.ts index 0ff658adfbdb..a6f70809ffd7 100644 --- a/packages/rule-tester/src/RuleTester.ts +++ b/packages/rule-tester/src/RuleTester.ts @@ -20,6 +20,7 @@ import { Linter } from '@typescript-eslint/utils/ts-eslint'; import { SourceCode } from 'eslint'; import merge from 'lodash.merge'; +import { prepareErrorsForSnapshot } from './snapshot/renderErrors'; import { TestFramework } from './TestFramework'; import type { InvalidTestCase, @@ -57,6 +58,32 @@ const ajv = ajvBuilder({ strictDefaults: true }); const TYPESCRIPT_ESLINT_PARSER = '@typescript-eslint/parser'; const DUPLICATE_PARSER_ERROR_MESSAGE = `Do not set the parser at the test level unless you want to use a parser other than "${TYPESCRIPT_ESLINT_PARSER}"`; +function memoizeRunRuleForItem< + TArg2 extends object, + TArg3 extends object, + TReturn extends object, +>( + fn: (arg1: string, arg2: TArg2, arg3: TArg3) => TReturn, +): (arg1: string, arg2: TArg2, arg3: TArg3) => TReturn { + const cache = new WeakMap>(); + return (arg1, arg2, arg3) => { + const cachedResult = cache.get(arg2)?.get(arg3); + if (cachedResult != null) { + return cachedResult; + } + + const result = fn(arg1, arg2, arg3); + let cache1 = cache.get(arg2); + if (!cache1) { + cache1 = new WeakMap(); + cache.set(arg2, cache1); + } + cache1.set(arg3, result); + + return result; + }; +} + /* * testerDefaultConfig must not be modified as it allows to reset the tester to * the initial default configuration @@ -76,6 +103,9 @@ export class RuleTester extends TestFramework { readonly #rules: Record = {}; readonly #linter: Linter = new Linter(); + // @ts-expect-error -- TS loosely types this as "Function" + declare 'constructor': typeof RuleTester; + /** * Creates a new instance of RuleTester. */ @@ -96,8 +126,7 @@ export class RuleTester extends TestFramework { // make sure that the parser doesn't hold onto file handles between tests // on linux (i.e. our CI env), there can be very a limited number of watch handles available - const constructor = this.constructor as typeof RuleTester; - constructor.afterAll(() => { + this.constructor.afterAll(() => { try { // instead of creating a hard dependency, just use a soft require // a bit weird, but if they're using this tooling, it'll be installed @@ -310,8 +339,6 @@ export class RuleTester extends TestFramework { rule: RuleModule, test: RunTests, ): void { - const constructor = this.constructor as typeof RuleTester; - if ( this.#testerConfig.dependencyConstraints && !satisfiesAllDependencyConstraints( @@ -320,8 +347,8 @@ export class RuleTester extends TestFramework { ) { // for frameworks like mocha or jest that have a "skip" version of their function // we can provide a nice skipped test! - constructor.describeSkip(ruleName, () => { - constructor.it( + this.constructor.describeSkip(ruleName, () => { + this.constructor.it( 'All tests skipped due to unsatisfied constructor dependency constraints', () => { // some frameworks error if there are no assertions @@ -396,8 +423,8 @@ export class RuleTester extends TestFramework { * This creates a test suite and pipes all supplied info through * one of the templates above. */ - constructor.describe(ruleName, () => { - constructor.describe('valid', () => { + this.constructor.describe(ruleName, () => { + this.constructor.describe('valid', () => { normalizedTests.valid.forEach(valid => { const testName = ((): string => { if (valid.name == null || valid.name.length === 0) { @@ -405,13 +432,13 @@ export class RuleTester extends TestFramework { } return valid.name; })(); - constructor[getTestMethod(valid)](sanitize(testName), () => { + this.constructor[getTestMethod(valid)](sanitize(testName), () => { this.#testValidTemplate(ruleName, rule, valid); }); }); }); - constructor.describe('invalid', () => { + this.constructor.describe('invalid', () => { normalizedTests.invalid.forEach(invalid => { const name = ((): string => { if (invalid.name == null || invalid.name.length === 0) { @@ -419,10 +446,35 @@ export class RuleTester extends TestFramework { } return invalid.name; })(); - constructor[getTestMethod(invalid)](sanitize(name), () => { + this.constructor[getTestMethod(invalid)](sanitize(name), () => { this.#testInvalidTemplate(ruleName, rule, invalid); }); }); + + const snapshotConfig = this.#testerConfig.snapshots; + if (snapshotConfig) { + this.constructor.describe('snapshots', () => { + const snapshotInfos = normalizedTests.invalid.map((test, index) => { + const result = this.runRuleForItem(ruleName, rule, test); + return prepareErrorsForSnapshot( + test.name ?? `Test #${index + 1}`, + test.filename ?? '', + test, + result.hasFixers ? result.output : null, + result.messages, + ); + }); + + this.constructor.expectSnapshot( + this.constructor, + { + snapshotBasePath: snapshotConfig.snapshotBasePath, + ruleName, + }, + snapshotInfos, + ); + }); + } }); }); } @@ -432,190 +484,201 @@ export class RuleTester extends TestFramework { * @throws {Error} If an invalid schema. * Use @private instead of #private to expose it for testing purposes */ - private runRuleForItem< - TMessageIds extends string, - TOptions extends readonly unknown[], - >( - ruleName: string, - rule: RuleModule, - item: ValidTestCase | InvalidTestCase, - ): { - messages: Linter.LintMessage[]; - output: string; - beforeAST: TSESTree.Program; - afterAST: TSESTree.Program; - } { - let config: TesterConfigWithDefaults = merge({}, this.#testerConfig); - let code; - let filename; - let output; - let beforeAST: TSESTree.Program; - let afterAST: TSESTree.Program; + private runRuleForItem = memoizeRunRuleForItem( + ( + ruleName: string, + rule: RuleModule, + item: + | ValidTestCase + | InvalidTestCase, + ): { + filename: string; + hasFixers: boolean; + messages: Linter.LintMessage[]; + output: string; + beforeAST: TSESTree.Program; + afterAST: TSESTree.Program; + } => { + let config: TesterConfigWithDefaults = merge({}, this.#testerConfig); + let code: string; + let filename = ''; + let hasFixers = false; + let output: string; + let beforeAST: TSESTree.Program; + let afterAST: TSESTree.Program; + + if (typeof item === 'string') { + code = item; + } else { + code = item.code; - if (typeof item === 'string') { - code = item; - } else { - code = item.code; + /* + * Assumes everything on the item is a config except for the + * parameters used by this tester + */ + const itemConfig: Record = { ...item }; - /* - * Assumes everything on the item is a config except for the - * parameters used by this tester - */ - const itemConfig: Record = { ...item }; + for (const parameter of RULE_TESTER_PARAMETERS) { + delete itemConfig[parameter]; + } - for (const parameter of RULE_TESTER_PARAMETERS) { - delete itemConfig[parameter]; + /* + * Create the config object from the tester config and this item + * specific configurations. + */ + config = merge(config, itemConfig); } - /* - * Create the config object from the tester config and this item - * specific configurations. - */ - config = merge(config, itemConfig); - } - - if (item.filename) { - filename = item.filename; - } - - if (hasOwnProperty(item, 'options')) { - assert(Array.isArray(item.options), 'options must be an array'); - if ( - item.options.length > 0 && - typeof rule === 'object' && - (!rule.meta || (rule.meta && rule.meta.schema == null)) - ) { - emitMissingSchemaWarning(ruleName); + if (item.filename) { + filename = item.filename; } - config.rules[ruleName] = ['error', ...item.options]; - } else { - config.rules[ruleName] = 'error'; - } - - const schema = getRuleOptionsSchema(rule); - - /* - * Setup AST getters. - * The goal is to check whether or not AST was modified when - * running the rule under test. - */ - this.#linter.defineRule('rule-tester/validate-ast', { - create() { - return { - Program(node): void { - beforeAST = cloneDeeplyExcludesParent(node); - }, - 'Program:exit'(node): void { - afterAST = node; - }, - }; - }, - }); - - if (typeof config.parser === 'string') { - assert( - path.isAbsolute(config.parser), - 'Parsers provided as strings to RuleTester must be absolute paths', - ); - } else { - config.parser = require.resolve(TYPESCRIPT_ESLINT_PARSER); - } - - this.#linter.defineParser( - config.parser, - wrapParser(require(config.parser) as Linter.ParserModule), - ); - if (schema) { - ajv.validateSchema(schema); - - if (ajv.errors) { - const errors = ajv.errors - .map(error => { - const field = - error.dataPath[0] === '.' - ? error.dataPath.slice(1) - : error.dataPath; - - return `\t${field}: ${error.message}`; - }) - .join('\n'); - - throw new Error( - [`Schema for rule ${ruleName} is invalid:`, errors].join( - // no space after comma to match eslint core - ',', - ), - ); + if (hasOwnProperty(item, 'options')) { + assert(Array.isArray(item.options), 'options must be an array'); + if ( + item.options.length > 0 && + typeof rule === 'object' && + (!rule.meta || (rule.meta && rule.meta.schema == null)) + ) { + emitMissingSchemaWarning(ruleName); + } + config.rules[ruleName] = [ + 'error', + ...(item.options as readonly unknown[]), + ]; + } else { + config.rules[ruleName] = 'error'; } + const schema = getRuleOptionsSchema(rule); + /* - * `ajv.validateSchema` checks for errors in the structure of the schema (by comparing the schema against a "meta-schema"), - * and it reports those errors individually. However, there are other types of schema errors that only occur when compiling - * the schema (e.g. using invalid defaults in a schema), and only one of these errors can be reported at a time. As a result, - * the schema is compiled here separately from checking for `validateSchema` errors. + * Setup AST getters. + * The goal is to check whether or not AST was modified when + * running the rule under test. */ - try { - ajv.compile(schema); - } catch (err) { - throw new Error( - `Schema for rule ${ruleName} is invalid: ${(err as Error).message}`, + this.#linter.defineRule('rule-tester/validate-ast', { + create() { + return { + Program(node): void { + beforeAST = cloneDeeplyExcludesParent(node); + }, + 'Program:exit'(node): void { + afterAST = node; + }, + }; + }, + }); + + if (typeof config.parser === 'string') { + assert( + path.isAbsolute(config.parser), + 'Parsers provided as strings to RuleTester must be absolute paths', ); + } else { + config.parser = require.resolve(TYPESCRIPT_ESLINT_PARSER); } - } - validate(config, 'rule-tester', id => (id === ruleName ? rule : null)); + this.#linter.defineParser( + config.parser, + wrapParser(require(config.parser) as Linter.ParserModule), + ); - // Verify the code. - // @ts-expect-error -- we don't define deprecated members on our types - const { getComments } = SourceCode.prototype as { getComments: unknown }; - let messages; + if (schema) { + ajv.validateSchema(schema); + + if (ajv.errors) { + const errors = ajv.errors + .map(error => { + const field = + error.dataPath[0] === '.' + ? error.dataPath.slice(1) + : error.dataPath; + + return `\t${field}: ${error.message}`; + }) + .join('\n'); + + throw new Error( + [`Schema for rule ${ruleName} is invalid:`, errors].join( + // no space after comma to match eslint core + ',', + ), + ); + } - try { - // @ts-expect-error -- we don't define deprecated members on our types - SourceCode.prototype.getComments = getCommentsDeprecation; - messages = this.#linter.verify(code, config, filename); - } finally { - // @ts-expect-error -- we don't define deprecated members on our types - SourceCode.prototype.getComments = getComments; - } + /* + * `ajv.validateSchema` checks for errors in the structure of the schema (by comparing the schema against a "meta-schema"), + * and it reports those errors individually. However, there are other types of schema errors that only occur when compiling + * the schema (e.g. using invalid defaults in a schema), and only one of these errors can be reported at a time. As a result, + * the schema is compiled here separately from checking for `validateSchema` errors. + */ + try { + ajv.compile(schema); + } catch (err) { + throw new Error( + `Schema for rule ${ruleName} is invalid: ${(err as Error).message}`, + ); + } + } - const fatalErrorMessage = messages.find(m => m.fatal); + validate(config, 'rule-tester', id => (id === ruleName ? rule : null)); - assert( - !fatalErrorMessage, - `A fatal parsing error occurred: ${fatalErrorMessage?.message}`, - ); + // Verify the code. + // @ts-expect-error -- we don't define deprecated members on our types + const { getComments } = SourceCode.prototype as { getComments: unknown }; + let messages; - // Verify if autofix makes a syntax error or not. - if (messages.some(m => m.fix)) { - output = SourceCodeFixer.applyFixes(code, messages).output; - const errorMessageInFix = this.#linter - .verify(output, config, filename) - .find(m => m.fatal); + try { + // @ts-expect-error -- we don't define deprecated members on our types + SourceCode.prototype.getComments = getCommentsDeprecation; + messages = this.#linter.verify(code, config, filename); + } finally { + // @ts-expect-error -- we don't define deprecated members on our types + SourceCode.prototype.getComments = getComments; + } + + const fatalErrorMessage = messages.find(m => m.fatal); assert( - !errorMessageInFix, - [ - 'A fatal parsing error occurred in autofix.', - `Error: ${errorMessageInFix?.message}`, - 'Autofix output:', - output, - ].join('\n'), + !fatalErrorMessage, + `A fatal parsing error occurred: ${fatalErrorMessage?.message}`, ); - } else { - output = code; - } - return { - messages, - output, - // is definitely assigned within the `rule-tester/validate-ast` rule - beforeAST: beforeAST!, - // is definitely assigned within the `rule-tester/validate-ast` rule - afterAST: cloneDeeplyExcludesParent(afterAST!), - }; - } + // Verify if autofix makes a syntax error or not. + if (messages.some(m => m.fix)) { + hasFixers = true; + output = SourceCodeFixer.applyFixes(code, messages).output; + const errorMessageInFix = this.#linter + .verify(output, config, filename) + .find(m => m.fatal); + + assert( + !errorMessageInFix, + [ + 'A fatal parsing error occurred in autofix.', + `Error: ${errorMessageInFix?.message}`, + 'Autofix output:', + output, + ].join('\n'), + ); + } else { + hasFixers = false; + output = code; + } + + return { + messages, + output, + hasFixers, + filename, + // is definitely assigned within the `rule-tester/validate-ast` rule + beforeAST: beforeAST!, + // is definitely assigned within the `rule-tester/validate-ast` rule + afterAST: cloneDeeplyExcludesParent(afterAST!), + }; + }, + ); /** * Check if the template is valid or not diff --git a/packages/rule-tester/src/TestFramework.ts b/packages/rule-tester/src/TestFramework.ts index dea77d746249..7e3bf950326c 100644 --- a/packages/rule-tester/src/TestFramework.ts +++ b/packages/rule-tester/src/TestFramework.ts @@ -1,3 +1,8 @@ +import { isCI } from 'std-env'; + +import type { RuleTesterExpectSnapshotFunction } from './snapshot/expectSnapshot'; +import { expectSnapshot } from './snapshot/expectSnapshot'; + /** * @param text a string describing the rule * @param callback the test callback @@ -24,6 +29,10 @@ export type RuleTesterTestFrameworkItFunction = */ skip?: RuleTesterTestFrameworkFunctionBase; }; +export type * from './snapshot/expectSnapshot'; +export type RuleTesterFrameworkSnapshotUpdateType = 'new' | 'all' | 'none'; +export type RuleTesterFrameworkShouldUpdateSnapshot = + () => RuleTesterFrameworkSnapshotUpdateType; type Maybe = T | null | undefined; @@ -38,6 +47,9 @@ let OVERRIDE_DESCRIBE_SKIP: Maybe = null; let OVERRIDE_IT: Maybe = null; let OVERRIDE_IT_ONLY: Maybe = null; let OVERRIDE_IT_SKIP: Maybe = null; +let OVERRIDE_EXPECT_SNAPSHOT: Maybe = null; +let OVERRIDE_GET_SHOULD_UPDATE_SNAPSHOT: Maybe = + null; /* * NOTE - If people use `mocha test.js --watch` command, the test function @@ -217,4 +229,57 @@ export abstract class TestFramework { static set itSkip(value: Maybe) { OVERRIDE_IT_SKIP = value; } + + static get expectSnapshot(): RuleTesterExpectSnapshotFunction { + if (OVERRIDE_EXPECT_SNAPSHOT != null) { + return OVERRIDE_EXPECT_SNAPSHOT; + } + + return expectSnapshot; + } + + static set expectSnapshot(value: Maybe) { + OVERRIDE_EXPECT_SNAPSHOT = value; + } + + static get getShouldUpdateSnapshots(): RuleTesterFrameworkShouldUpdateSnapshot { + if (OVERRIDE_GET_SHOULD_UPDATE_SNAPSHOT != null) { + return OVERRIDE_GET_SHOULD_UPDATE_SNAPSHOT; + } + + return (): RuleTesterFrameworkSnapshotUpdateType => { + if (isCI) { + // never update snapshots on CI + return 'none'; + } + + if (process.env.UPDATE_SNAPSHOT != null) { + return process.env.UPDATE_SNAPSHOT !== '0' && + process.env.UPDATE_SNAPSHOT !== 'false' + ? 'new' + : 'all'; + } + + if (typeof expect !== 'undefined') { + // https://github.com/jestjs/jest/blob/bc26cd79e60b7bf29d854293f0f011936fda1a5a/packages/jest-snapshot/src/State.ts#L59 + // this should also work for vitest as it maintains compat with jest + const maybeJestState = ( + expect.getState().snapshotState as { + _updateSnapshot?: RuleTesterFrameworkSnapshotUpdateType; + } + )?._updateSnapshot; + if (maybeJestState != null) { + return maybeJestState; + } + } + + return 'new'; + }; + } + + static set getShouldUpdateSnapshots( + value: Maybe, + ) { + OVERRIDE_GET_SHOULD_UPDATE_SNAPSHOT = value; + } } diff --git a/packages/rule-tester/src/snapshot/expectSnapshot.ts b/packages/rule-tester/src/snapshot/expectSnapshot.ts new file mode 100644 index 000000000000..6fc8667b5ad8 --- /dev/null +++ b/packages/rule-tester/src/snapshot/expectSnapshot.ts @@ -0,0 +1,234 @@ +import assert from 'node:assert'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import zlib from 'node:zlib'; + +import makeDir from 'make-dir'; + +import type { TestFramework } from '../TestFramework'; +import type { SingleTestResult, TestReportSuggestion } from './types'; + +/** + * @param testFramework the test framework used by the RuleTester + * @param pathInfo information about the path that snapshots might be written to + * @param results the snapshot results collected from the tests + */ +export type RuleTesterExpectSnapshotFunction = ( + testFramework: typeof TestFramework, + pathInfo: { + /** + * The base folder that snapshots should be written within + */ + readonly snapshotBasePath: string; + /** + * The name of the rule being tested as provided to the `.run` method + */ + readonly ruleName: string; + }, + results: readonly SingleTestResult[], +) => void; + +function hasErrorCode(e: unknown): e is { code: unknown } { + return typeof e === 'object' && e != null && 'code' in e; +} + +/** + * Default snapshot tester which writes a single snapshot file for the test + */ +export const expectSnapshot: RuleTesterExpectSnapshotFunction = ( + testFramework, + { snapshotBasePath, ruleName }, + results, +): void => { + if (results.length === 0) { + return; + } + + const snapshotFolder = path.resolve(snapshotBasePath); + try { + makeDir.sync(snapshotFolder); + } catch (e) { + if (hasErrorCode(e) && e.code === 'EEXIST') { + // already exists - ignored + } else { + throw e; + } + } + const snapshotMdFilePath = path.join( + snapshotFolder, + `${ruleName}.snapshot.md`, + ); + const snapshotMetaFilePath = path.join( + snapshotFolder, + `${ruleName}.snapshotstate`, + ); + + const existingSnapshotMeta = readSnapshotMeta(snapshotMetaFilePath); + const updateSnapshots = testFramework.getShouldUpdateSnapshots(); + + let passCount = 0; + const newResults = new Set(); + for (const result of results) { + testFramework.it(result.testName, () => { + const existingResult = existingSnapshotMeta[result.testHash]; + if (existingResult == null) { + if (updateSnapshots === 'none') { + assert.fail('No existing result for test'); + } + + assert.ok(true, 'No existing result for test'); + passCount += 1; + newResults.add(result); + return; + } + + if (updateSnapshots === 'all') { + assert.ok(true, 'All snapshots are being force-updated'); + passCount += 1; + return; + } + + // we could just do `assert.deepStrictEquals` here, but for better DevX we + // don't want to expose users to our internal object representation, so + // instead we granularly assert everything with clear + + assert.strictEqual( + result.fixOutput, + existingResult.fixOutput, + 'The fix output has changed.', + ); + assert.strictEqual( + result.errors.length, + existingResult.errors.length, + 'The number of reported errors has changed.', + ); + + for ( + let errorIndex = 0; + errorIndex < result.errors.length; + errorIndex += 1 + ) { + const error = result.errors[errorIndex]; + const existingError = existingResult.errors[errorIndex]; + assert.strictEqual( + error.errorCodeFrame, + existingError.errorCodeFrame, + `The error at index ${errorIndex} has changed.`, + ); + + assert.strictEqual( + error.suggestions?.length, + existingError.suggestions?.length, + 'The number of reported suggestions has changed.', + ); + + if (error.suggestions != null && existingError.suggestions != null) { + for ( + let suggestionIndex = 0; + suggestionIndex < error.suggestions.length; + suggestionIndex += 1 + ) { + // TS can't resolve the type here for whatever reason + const suggestion: TestReportSuggestion = + error.suggestions[suggestionIndex]; + const existingSuggestion: TestReportSuggestion = + existingError.suggestions[suggestionIndex]; + + assert.strictEqual( + suggestion.message, + existingSuggestion.message, + `The message of the suggestion at index ${errorIndex} has changed.`, + ); + assert.strictEqual( + suggestion.fixOutput, + existingSuggestion.fixOutput, + `The fix output of the suggestion at index ${errorIndex} has changed.`, + ); + } + } + } + + passCount += 1; + }); + } + + testFramework.afterAll(() => { + if (passCount !== results.length || updateSnapshots === 'none') { + return; + } + + writeSnapshotMeta( + snapshotMetaFilePath, + Object.fromEntries(results.map(r => [r.testHash, r])), + ); + writeMarkdown(ruleName, snapshotMdFilePath, results); + }); +}; + +type SnapshotMeta = Record; +function readSnapshotMeta(metaPath: string): SnapshotMeta { + try { + const raw = fs.readFileSync(metaPath); + const uncompressedBuf = zlib.gunzipSync(raw); + return JSON.parse(uncompressedBuf.toString()) as SnapshotMeta; + } catch { + return {}; + } +} +function writeSnapshotMeta(metaPath: string, blob: SnapshotMeta): void { + const buf = Buffer.from(JSON.stringify(blob)); + const compressed = zlib.gzipSync(buf); + fs.writeFileSync(metaPath, compressed); +} +function writeMarkdown( + ruleName: string, + markdownPath: string, + results: readonly SingleTestResult[], +): void { + const markdownLines = [`# ${ruleName}`, '']; + + for (const result of results) { + const codeFence = (code: string): string => + [ + '', + `\`\`\`${result.testLanguageFlavour}`, + code, + '```', + ].join('\n'); + + markdownLines.push( + `## ${result.testName}`, + '', + '### Test Code', + '', + codeFence(result.code), + '', + '### Fix Output', + '', + result.fixOutput == null ? 'No fix applied' : codeFence(result.fixOutput), + '', + '### Errors', + '', + ); + + for (const error of result.errors) { + markdownLines.push(codeFence(error.errorCodeFrame), ''); + + if (error.suggestions != null) { + markdownLines.push('#### Suggestions', ''); + for (const suggestion of error.suggestions ?? []) { + markdownLines.push( + `##### ${suggestion.message}`, + '', + codeFence(suggestion.fixOutput), + ); + } + } + } + } + + // include a trailing newline + markdownLines.push(''); + + fs.writeFileSync(markdownPath, markdownLines.join('\n'), 'utf8'); +} diff --git a/packages/rule-tester/src/snapshot/renderErrors.ts b/packages/rule-tester/src/snapshot/renderErrors.ts new file mode 100644 index 000000000000..dcb8ed7e1c36 --- /dev/null +++ b/packages/rule-tester/src/snapshot/renderErrors.ts @@ -0,0 +1,86 @@ +import * as crypto from 'node:crypto'; +import * as path from 'node:path'; + +import type { SourceLocation as BabelSourceLocation } from '@babel/code-frame'; +import { codeFrameColumns } from '@babel/code-frame'; +import type { Linter } from '@typescript-eslint/utils/ts-eslint'; + +import type { InvalidTestCase } from '../types/InvalidTestCase'; +import * as SourceCodeFixer from '../utils/SourceCodeFixer'; +import type { SingleTestResult, TestReportError } from './types'; + +function renderError(code: string, error: Linter.LintMessage): string { + const location: BabelSourceLocation = { + start: { + line: error.line, + column: error.column, + }, + }; + + if ( + typeof error.endLine === 'number' && + typeof error.endColumn === 'number' + ) { + location.end = { + line: error.endLine, + column: error.endColumn, + }; + } + + return codeFrameColumns(code, location, { + forceColor: false, + highlightCode: false, + linesAbove: Number.POSITIVE_INFINITY, + linesBelow: Number.POSITIVE_INFINITY, + message: error.message, + }); +} + +export function prepareErrorsForSnapshot( + testName: string, + testFilename: string, + test: InvalidTestCase, + output: string | null, + messages: Linter.LintMessage[], +): SingleTestResult { + const errors: TestReportError[] = []; + for (const message of messages) { + errors.push({ + errorCodeFrame: renderError(test.code, message), + suggestions: + message.suggestions == null + ? null + : message.suggestions.map(s => ({ + message: s.desc, + fixOutput: SourceCodeFixer.applyFixes(test.code, [s]).output, + })), + }); + } + + const testFilenameExtension = path.extname(testFilename).substring(1); + + const testHash = (() => { + const { + errors: _errors, + dependencyConstraints: _dependencyConstraints, + only: _only, + skip: _skip, + output: _output, + ...hashableProps + } = test; + return crypto + .createHash('sha256') + .update(JSON.stringify(hashableProps)) + .digest('base64'); + })(); + + return { + code: test.code, + errors, + fixOutput: output, + testLanguageFlavour: + testFilenameExtension === '' ? 'ts' : testFilenameExtension, + testName, + testHash, + }; +} diff --git a/packages/rule-tester/src/snapshot/types.ts b/packages/rule-tester/src/snapshot/types.ts new file mode 100644 index 000000000000..6d6249d00709 --- /dev/null +++ b/packages/rule-tester/src/snapshot/types.ts @@ -0,0 +1,48 @@ +export interface TestReportSuggestion { + /** + * The hydrated suggestion message + */ + readonly message: string; + /** + * The results of applying the suggestion fixer to the code. + */ + readonly fixOutput: string; +} +export interface TestReportError { + /** + * The code string with the error rendered in it. + */ + readonly errorCodeFrame: string; + /** + * Information about suggestions provided alongside the error. + */ + readonly suggestions: TestReportSuggestion[] | null; +} +export interface SingleTestResult { + /** + * The original test code. + */ + readonly code: string; + /** + * The results of applying the autofixers to the code. + * `null` if there was no fixers applied. + */ + readonly fixOutput: string | null; + /** + * The errors reported during the test run. + */ + readonly errors: readonly TestReportError[]; + /** + * The unique hash for the test so that it can be uniquely keyed in an object + */ + readonly testHash: string; + /** + * The resolved name of the test. + */ + readonly testName: string; + /** + * The language flavour of the test code for rendering markdown code fences + * with appropriate syntax highlighting + */ + readonly testLanguageFlavour: string; +} diff --git a/packages/rule-tester/src/types/RuleTesterConfig.ts b/packages/rule-tester/src/types/RuleTesterConfig.ts index c722c5be074e..ee88294fe96b 100644 --- a/packages/rule-tester/src/types/RuleTesterConfig.ts +++ b/packages/rule-tester/src/types/RuleTesterConfig.ts @@ -2,6 +2,7 @@ import type { Linter, ParserOptions } from '@typescript-eslint/utils/ts-eslint'; import type { DependencyConstraint } from './DependencyConstraint'; +// need to keep this in sync with the schema in `config-schema.ts` export interface RuleTesterConfig extends Linter.Config { /** * The default parser to use for tests. @@ -24,4 +25,16 @@ export interface RuleTesterConfig extends Linter.Config { ts: string; tsx: string; }>; + /** + * Whether or not to do snapshot testing for "invalid" test cases. + * @default false + */ + readonly snapshots?: + | false + | { + /** + * The absolute path to use as the basis for writing snapshots. + */ + readonly snapshotBasePath: string; + }; } diff --git a/packages/rule-tester/src/utils/config-schema.ts b/packages/rule-tester/src/utils/config-schema.ts index 8261ac8749c8..5be93d7e3553 100644 --- a/packages/rule-tester/src/utils/config-schema.ts +++ b/packages/rule-tester/src/utils/config-schema.ts @@ -2,6 +2,7 @@ import type { JSONSchema } from '@typescript-eslint/utils'; +// need to keep this in sync with the types in `RuleTesterConfig.ts` const baseConfigProperties: JSONSchema.JSONSchema4['properties'] = { $schema: { type: 'string' }, defaultFilenames: { @@ -35,6 +36,24 @@ const baseConfigProperties: JSONSchema.JSONSchema4['properties'] = { reportUnusedDisableDirectives: { type: 'boolean' }, rules: { type: 'object' }, settings: { type: 'object' }, + snapshots: { + oneOf: [ + { + type: 'object', + properties: { + snapshotBasePath: { + type: 'string', + }, + }, + additionalProperties: false, + required: ['snapshotBasePath'], + }, + { + type: 'boolean', + enum: [false], + }, + ], + }, ecmaFeatures: { type: 'object' }, // deprecated; logs a warning when used }; diff --git a/yarn.lock b/yarn.lock index a73bc274340e..cbdd87f626f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13714,6 +13714,11 @@ std-env@^3.0.1: resolved "https://registry.yarnpkg.com/std-env/-/std-env-3.0.1.tgz#bc4cbc0e438610197e34c2d79c3df30b491f5182" integrity sha512-mC1Ps9l77/97qeOZc+HrOL7TIaOboHqMZ24dGVQrlxFcpPpfCHpH+qfUT7Dz+6mlG8+JPA1KfBQo19iC/+Ngcw== +std-env@^3.3.3: + version "3.3.3" + resolved "https://registry.yarnpkg.com/std-env/-/std-env-3.3.3.tgz#a54f06eb245fdcfef53d56f3c0251f1d5c3d01fe" + integrity sha512-Rz6yejtVyWnVjC1RFvNmYL10kgjC49EOghxWn0RFqlCHGFpQx+Xe7yW3I4ceK1SGrWIGMjD5Kbue8W/udkbMJg== + stop-iteration-iterator@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz#6a60be0b4ee757d1ed5254858ec66b10c49285e4" From ebc87459a5e35bed08a61549f6c7585442d10b44 Mon Sep 17 00:00:00 2001 From: Brad Zacher Date: Sat, 6 May 2023 12:08:20 +0800 Subject: [PATCH 2/2] update ci config --- .github/workflows/ci.yml | 1 + .github/workflows/semantic-pr-titles.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5bebd18b49d..999ace34d97f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,7 @@ jobs: 'parser', 'repo-tools', 'rule-schema-to-typescript-types', + 'rule-tester', 'scope-manager', 'type-utils', 'typescript-estree', diff --git a/.github/workflows/semantic-pr-titles.yml b/.github/workflows/semantic-pr-titles.yml index fdbc928068c1..9bfafeac010f 100644 --- a/.github/workflows/semantic-pr-titles.yml +++ b/.github/workflows/semantic-pr-titles.yml @@ -31,6 +31,9 @@ jobs: eslint-plugin-internal eslint-plugin-tslint parser + repo-tools + rule-schema-to-typescript-types + rule-tester scope-manager type-utils types