✨ Compile doc code fences into tests
Every `ts`-tagged fence in README.md / CONTRIBUTING.md now becomes an executed `node:test` case in a gitignored generated file, so a documented example cannot drift from the API. The generator hoists and merges the leading imports, rewrites the library specifier to the `#test-tiny-pattern-ts` alias, and rejects a fence with no describing paragraph or one that re-imports a prelude module. CI regenerates via `pretest:ci`; `create:doc-tests` runs the generator, formats, then typechecks the output against a scoped tsconfig that relaxes `noUnusedLocals`. c8's default excludes already omit the generated `*.test.ts`, so `test:ci` is unchanged.
This commit is contained in:
tmu committed
1 parent
d1963b0329
commit
30d97a9209
15 files changed
+1180
-8
No files matched your search
@@ -0,0 +1,445 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import type { Code, Heading, Paragraph, PhrasingContent, Root } from "mdast";
|
||||
import { fromMarkdown } from "mdast-util-from-markdown";
|
||||
|
||||
/**
|
||||
* Compile the TypeScript examples embedded in the prose docs into runnable
|
||||
* tests, so a doc fence that drifts from the API fails CI. Every ```ts fence in
|
||||
* the scanned Markdown becomes a `test(...)` in a generated
|
||||
* `src/doc-test/__generated__/<Name>.test.ts`; the file is typechecked by
|
||||
* `tsc` and executed by `node --test` exactly like a hand-written test.
|
||||
*
|
||||
* Usage: node --strip-types scripts/create-doc-tests.ts
|
||||
*
|
||||
* Why regenerate-then-typecheck instead of a lint plugin: development/docs.md §
|
||||
* Validating Markdown code fences. The repo is TypeScript 7 (the native
|
||||
* compiler), which does not expose the legacy JS compiler API, so the only
|
||||
* faithful check is to emit real `.ts` files and let the existing `check:tsc` /
|
||||
* `node --test` pipeline judge them.
|
||||
*/
|
||||
|
||||
/** The package's public entry, as spelled inside the examples. */
|
||||
const LIBRARY_SOURCE = "tiny-pattern-ts";
|
||||
|
||||
/** Matches the library specifier (single- or double-quoted) inside an import. */
|
||||
const LIBRARY_SPECIFIER = new RegExp(
|
||||
`(?<quote>["'])${LIBRARY_SOURCE}\\k<quote>`,
|
||||
"g",
|
||||
);
|
||||
|
||||
/**
|
||||
* A source specifier resolved to `src/index.ts` by `package.json#imports`.
|
||||
* Examples import `from "tiny-pattern-ts"`, which neither `node` nor `tsc`
|
||||
* resolves to source before `dist/` exists, so the generator rewrites it.
|
||||
*/
|
||||
const PRELUDE_IMPORT_SOURCE = "#test-tiny-pattern-ts";
|
||||
|
||||
/** Markdown files to scan, relative to the repo root. */
|
||||
const SOURCES = ["README.md", "CONTRIBUTING.md"];
|
||||
|
||||
/** Directory the generated tests are written to (gitignored contents). */
|
||||
const OUTPUT_DIR = "src/doc-test/__generated__";
|
||||
|
||||
/** Fences whose info-string matches one of these are compiled; others skipped. */
|
||||
const TYPESCRIPT_LANGS: ReadonlySet<string> = new Set(["ts", "typescript"]);
|
||||
|
||||
/**
|
||||
* Modules the generated file already imports. An example that imports one of
|
||||
* these would collide with the prelude binding (duplicate `test` / `assert`), so
|
||||
* it is surfaced as a fatal error and the example is rewritten.
|
||||
*/
|
||||
const PRELUDE_MODULES: ReadonlySet<string> = new Set([
|
||||
"node:assert",
|
||||
"node:test",
|
||||
]);
|
||||
|
||||
/** One line of the prelude every generated file starts with. */
|
||||
const PRELUDE_TEST = 'import { test } from "node:test";';
|
||||
const PRELUDE_ASSERT = 'import { strict as assert } from "node:assert";';
|
||||
|
||||
/** Indentation applied to every fence body line inside the `test` callback. */
|
||||
const INDENT = " ";
|
||||
|
||||
const EMPTY = "";
|
||||
const INITIAL_COUNT = 0;
|
||||
const WRITE_INCREMENT = 1;
|
||||
const FIRST_ITEM = 0;
|
||||
const FAILURE_EXIT_CODE = 1;
|
||||
|
||||
/** Strip all leading blank lines so the import scan starts at real content. */
|
||||
const LEADING_BLANK_LINES = /^(?:[ \t]*\r?\n)+/;
|
||||
/** Collapse the plain text of a title paragraph to one line. */
|
||||
const WHITESPACE = /\s+/g;
|
||||
/** Split an info-string like `ts title="x"` into its bare language. */
|
||||
const LANG_SEPARATOR = /\s+/;
|
||||
|
||||
/** One leading `import` statement, including a multi-line named import. */
|
||||
const IMPORT_STATEMENT =
|
||||
/^import\s+(?:(?:type\s+)?[\w$*{}\s,]+?\s+from\s+)?["'][^"'\n]+["']\s*;?[ \t]*(?:\r?\n|$)/;
|
||||
|
||||
/** `import { a, b } from "src";`, with an optional `type` keyword. */
|
||||
const NAMED_IMPORT =
|
||||
/^import\s+(?<isType>type\s+)?\{(?<specifiers>[^}]*)\}\s+from\s+["'](?<source>[^"']+)["']\s*;?$/;
|
||||
|
||||
/** The module in a `… from "src"` statement. */
|
||||
const FROM_SOURCE = /from\s+["'](?<source>[^"']+)["']/;
|
||||
|
||||
/** The module in a side-effect `import "src"` statement. */
|
||||
const SIDE_EFFECT_SOURCE = /^import\s+["'](?<source>[^"']+)["']/;
|
||||
|
||||
type Block = Root["children"][number];
|
||||
|
||||
class DocTestError extends Error {
|
||||
public constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "DocTestError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Merge named specifiers that target the same module into one import. */
|
||||
interface NamedImportGroup {
|
||||
readonly source: string;
|
||||
readonly isType: boolean;
|
||||
readonly names: string[];
|
||||
}
|
||||
|
||||
/** One compiled fence, ready to be wrapped in a `test(...)`. */
|
||||
interface DocCase {
|
||||
readonly title: string;
|
||||
readonly body: string;
|
||||
}
|
||||
|
||||
/** Accumulator threaded through the Markdown walk. */
|
||||
interface BuilderState {
|
||||
readonly named: Map<string, NamedImportGroup>;
|
||||
readonly passthrough: Set<string>;
|
||||
readonly cases: DocCase[];
|
||||
title: string | undefined;
|
||||
}
|
||||
|
||||
/** Read one capture group, tolerating the type-level `groups` optionality. */
|
||||
const groupOf = (match: RegExpExecArray, name: string): string | undefined => {
|
||||
const { groups } = match;
|
||||
return groups === undefined ? undefined : groups[name];
|
||||
};
|
||||
|
||||
/** Recursively concatenate the literal text of a single inline node. */
|
||||
const inlineText = (node: PhrasingContent): string => {
|
||||
if ("value" in node) {
|
||||
return node.value;
|
||||
}
|
||||
if ("children" in node) {
|
||||
return node.children.map(inlineText).join(EMPTY);
|
||||
}
|
||||
return EMPTY;
|
||||
};
|
||||
|
||||
/** Render a paragraph/heading node to its plain text for use as a test title. */
|
||||
const textOf = (node: Paragraph | Heading): string =>
|
||||
node.children.map(inlineText).join(EMPTY).replace(WHITESPACE, " ").trim();
|
||||
|
||||
/**
|
||||
* Split a fence into its leading `import` statements and the executable body.
|
||||
* Only a contiguous run of imports at the very top is hoisted; anything after
|
||||
* the first non-import line stays in the body verbatim.
|
||||
*/
|
||||
const splitImports = (code: string): { imports: string[]; body: string } => {
|
||||
const imports: string[] = [];
|
||||
let rest = code.replace(LEADING_BLANK_LINES, EMPTY);
|
||||
let match = IMPORT_STATEMENT.exec(rest);
|
||||
while (rest.startsWith("import") && match !== null) {
|
||||
const statement = match[FIRST_ITEM] ?? EMPTY;
|
||||
imports.push(statement.trim());
|
||||
rest = rest.slice(statement.length).replace(LEADING_BLANK_LINES, EMPTY);
|
||||
match = IMPORT_STATEMENT.exec(rest);
|
||||
}
|
||||
return { imports, body: rest.trim() };
|
||||
};
|
||||
|
||||
/** The module an import statement points at, or `undefined` if unreadable. */
|
||||
const importSource = (statement: string): string | undefined => {
|
||||
const fromMatch = FROM_SOURCE.exec(statement);
|
||||
if (fromMatch !== null) {
|
||||
return groupOf(fromMatch, "source");
|
||||
}
|
||||
const sideEffectMatch = SIDE_EFFECT_SOURCE.exec(statement);
|
||||
return sideEffectMatch === null
|
||||
? undefined
|
||||
: groupOf(sideEffectMatch, "source");
|
||||
};
|
||||
|
||||
/**
|
||||
* Reject an example that imports a harness-provided module, then rewrite the
|
||||
* library specifier to the source alias so the emitted file resolves.
|
||||
*/
|
||||
const normalizeImport = (statement: string): string => {
|
||||
const source = importSource(statement);
|
||||
if (source !== undefined && PRELUDE_MODULES.has(source)) {
|
||||
throw new DocTestError(
|
||||
`example imports "${source}", which the harness injects; remove the line:\n ${statement.trim()}`,
|
||||
);
|
||||
}
|
||||
return statement.replace(
|
||||
LIBRARY_SPECIFIER,
|
||||
`$<quote>${PRELUDE_IMPORT_SOURCE}$<quote>`,
|
||||
);
|
||||
};
|
||||
|
||||
/** Split the `{ a, b }` contents of a named import into trimmed specifiers. */
|
||||
const specifiersOf = (raw: string): string[] =>
|
||||
raw
|
||||
.split(",")
|
||||
.map((name) => name.trim())
|
||||
.filter((name) => name.length > INITIAL_COUNT);
|
||||
|
||||
/** Add named specifiers to the group for `(isType, source)`, merging. */
|
||||
const addNamedImport = (
|
||||
named: Map<string, NamedImportGroup>,
|
||||
ref: Pick<NamedImportGroup, "source" | "isType">,
|
||||
names: readonly string[],
|
||||
): void => {
|
||||
const key = `${ref.isType ? "type:" : "value:"}${ref.source}`;
|
||||
const existing = named.get(key);
|
||||
if (existing === undefined) {
|
||||
named.set(key, {
|
||||
source: ref.source,
|
||||
isType: ref.isType,
|
||||
names: [...names],
|
||||
});
|
||||
return;
|
||||
}
|
||||
for (const name of names) {
|
||||
if (!existing.names.includes(name)) {
|
||||
existing.names.push(name);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Sort each hoisted import into a merged named group or a passthrough set. */
|
||||
const collectImports = (
|
||||
statements: readonly string[],
|
||||
named: Map<string, NamedImportGroup>,
|
||||
passthrough: Set<string>,
|
||||
): void => {
|
||||
for (const raw of statements) {
|
||||
const statement = normalizeImport(raw);
|
||||
const match = NAMED_IMPORT.exec(statement);
|
||||
if (match === null) {
|
||||
passthrough.add(statement);
|
||||
} else {
|
||||
addNamedImport(
|
||||
named,
|
||||
{
|
||||
source: groupOf(match, "source") ?? EMPTY,
|
||||
isType: groupOf(match, "isType") !== undefined,
|
||||
},
|
||||
specifiersOf(groupOf(match, "specifiers") ?? EMPTY),
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Render the hoisted imports: merged named groups first, then the rest. */
|
||||
const renderImports = (
|
||||
named: ReadonlyMap<string, NamedImportGroup>,
|
||||
passthrough: ReadonlySet<string>,
|
||||
): string[] => {
|
||||
const lines: string[] = [];
|
||||
for (const group of named.values()) {
|
||||
const keyword = group.isType ? "import type" : "import";
|
||||
lines.push(
|
||||
`${keyword} { ${group.names.join(", ")} } from "${group.source}";`,
|
||||
);
|
||||
}
|
||||
for (const statement of passthrough) {
|
||||
lines.push(statement);
|
||||
}
|
||||
return lines;
|
||||
};
|
||||
|
||||
/** The bare language of a fence's info-string, e.g. `ts` in `ts title="x"`. */
|
||||
const typescriptLang = (code: Code): string =>
|
||||
(code.lang ?? EMPTY).trim().split(LANG_SEPARATOR)[FIRST_ITEM] ?? EMPTY;
|
||||
|
||||
/** Set the title a following fence will inherit. */
|
||||
const handleTitleNode = (
|
||||
state: BuilderState,
|
||||
node: Paragraph | Heading,
|
||||
): void => {
|
||||
state.title = textOf(node);
|
||||
};
|
||||
|
||||
/** Report and reset a non-TS fence that was skipped. */
|
||||
const skipFence = (state: BuilderState, name: string, code: Code): void => {
|
||||
process.stderr.write(
|
||||
`${name}: skipped non-TypeScript fence (lang="${code.lang ?? EMPTY}")\n`,
|
||||
);
|
||||
state.title = undefined;
|
||||
};
|
||||
|
||||
/** Compile a described TS fence into a case, or reject it. */
|
||||
const compileFence = (
|
||||
state: BuilderState,
|
||||
name: string,
|
||||
code: Code,
|
||||
): DocCase => {
|
||||
const { title } = state;
|
||||
if (title === undefined) {
|
||||
const lang = typescriptLang(code);
|
||||
throw new DocTestError(
|
||||
`${name}: a \`\`\`${lang} fence has no preceding paragraph or ` +
|
||||
`heading to use as its test title — describe the example.`,
|
||||
);
|
||||
}
|
||||
const { imports, body } = splitImports(code.value);
|
||||
collectImports(imports, state.named, state.passthrough);
|
||||
return { title, body };
|
||||
};
|
||||
|
||||
/** Compile one TS fence into a case, or reject/skip it. */
|
||||
const handleCodeNode = (
|
||||
state: BuilderState,
|
||||
name: string,
|
||||
code: Code,
|
||||
): void => {
|
||||
const lang = typescriptLang(code);
|
||||
if (!TYPESCRIPT_LANGS.has(lang)) {
|
||||
skipFence(state, name, code);
|
||||
return;
|
||||
}
|
||||
state.cases.push(compileFence(state, name, code));
|
||||
};
|
||||
|
||||
/** Route one Markdown block, preserving the "described fence" invariant. */
|
||||
const handleNode = (state: BuilderState, name: string, node: Block): void => {
|
||||
if (node.type === "paragraph" || node.type === "heading") {
|
||||
handleTitleNode(state, node);
|
||||
} else if (node.type === "code") {
|
||||
handleCodeNode(state, name, node);
|
||||
} else {
|
||||
// Only a paragraph/heading introduces a fence; any other block breaks
|
||||
// the "immediately preceded" chain.
|
||||
state.title = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
/** Walk one Markdown file and collect its cases and hoisted imports. */
|
||||
const parseDoc = (
|
||||
name: string,
|
||||
markdown: string,
|
||||
): Omit<BuilderState, "title"> => {
|
||||
const state: BuilderState = {
|
||||
named: new Map(),
|
||||
passthrough: new Set(),
|
||||
cases: [],
|
||||
title: undefined,
|
||||
};
|
||||
for (const node of fromMarkdown(markdown).children) {
|
||||
handleNode(state, name, node);
|
||||
}
|
||||
return {
|
||||
named: state.named,
|
||||
passthrough: state.passthrough,
|
||||
cases: state.cases,
|
||||
};
|
||||
};
|
||||
|
||||
/** Indent a fence body one level for the body of the `test` callback. */
|
||||
const indentBody = (body: string): string =>
|
||||
body
|
||||
.split("\n")
|
||||
.map((line) =>
|
||||
line.length > INITIAL_COUNT ? `${INDENT}${line}` : line,
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
/** Wrap one compiled fence in an executed `test(...)`. */
|
||||
const renderCase = (docCase: DocCase): string => {
|
||||
const indented = indentBody(docCase.body);
|
||||
return `test(${JSON.stringify(docCase.title)}, () => {\n${indented}\n});`;
|
||||
};
|
||||
|
||||
/** Build the prelude + hoisted imports that top every generated file. */
|
||||
const buildHeader = (name: string, imports: readonly string[]): string[] => {
|
||||
const header = [
|
||||
"// Generated by scripts/create-doc-tests.ts — do not edit by hand.",
|
||||
`// Source: ${name}`,
|
||||
EMPTY,
|
||||
PRELUDE_TEST,
|
||||
PRELUDE_ASSERT,
|
||||
];
|
||||
if (imports.length > INITIAL_COUNT) {
|
||||
header.push(EMPTY, ...imports);
|
||||
}
|
||||
header.push(EMPTY);
|
||||
return header;
|
||||
};
|
||||
|
||||
/** Turn one Markdown file into the source of its generated test file. */
|
||||
const buildTestFile = (name: string, markdown: string): string => {
|
||||
const parsed = parseDoc(name, markdown);
|
||||
if (parsed.cases.length === INITIAL_COUNT) {
|
||||
return EMPTY;
|
||||
}
|
||||
const imports = renderImports(parsed.named, parsed.passthrough);
|
||||
const header = buildHeader(name, imports);
|
||||
const blocks = parsed.cases.map(renderCase);
|
||||
return `${header.join("\n")}\n${blocks.join("\n\n")}\n`;
|
||||
};
|
||||
|
||||
/** Map a source Markdown path to its generated test path under OUTPUT_DIR. */
|
||||
const outputFor = (source: string): string =>
|
||||
path.join(
|
||||
OUTPUT_DIR,
|
||||
`${path.basename(source, path.extname(source))}.test.ts`,
|
||||
);
|
||||
|
||||
/** Write a generated file when there is content, else clear a stale one. */
|
||||
const writeIfAny = (dest: string, name: string, out: string): boolean => {
|
||||
if (out === EMPTY) {
|
||||
process.stderr.write(`${name}: no TypeScript fences\n`);
|
||||
fs.rmSync(dest, { force: true });
|
||||
return false;
|
||||
}
|
||||
fs.writeFileSync(dest, out);
|
||||
return true;
|
||||
};
|
||||
|
||||
/** Generate the doc-test for one source; return whether a file was written. */
|
||||
const generateFor = (root: string, source: string): boolean => {
|
||||
const abs = path.join(root, source);
|
||||
if (!fs.existsSync(abs)) {
|
||||
process.stderr.write(`skipped missing ${source}\n`);
|
||||
return false;
|
||||
}
|
||||
const name = path.basename(source);
|
||||
const out = buildTestFile(name, fs.readFileSync(abs, "utf8"));
|
||||
return writeIfAny(path.join(root, outputFor(source)), name, out);
|
||||
};
|
||||
|
||||
const main = (): void => {
|
||||
const root = process.cwd();
|
||||
fs.mkdirSync(path.join(root, OUTPUT_DIR), { recursive: true });
|
||||
let written = INITIAL_COUNT;
|
||||
for (const source of SOURCES) {
|
||||
if (generateFor(root, source)) {
|
||||
written += WRITE_INCREMENT;
|
||||
}
|
||||
}
|
||||
process.stdout.write(
|
||||
`created doc-tests for ${written} file(s) under ${OUTPUT_DIR}/\n`,
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
main();
|
||||
} catch (error) {
|
||||
if (error instanceof DocTestError) {
|
||||
process.stderr.write(`${error.message}\n`);
|
||||
process.exitCode = FAILURE_EXIT_CODE;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user