Add dts bundling

Author: jakebailey

.gitignore

@@ -47,6 +47,7 @@ scripts/configurePrerelease.js
 scripts/configureLanguageServiceBuild.js
 scripts/open-user-pr.js
 scripts/open-cherry-pick-pr.js
+scripts/dtsBundler.js
 scripts/processDiagnosticMessages.d.ts
 scripts/processDiagnosticMessages.js
 scripts/produceLKG.js

Gulpfile.js

@@ -126,6 +126,22 @@ const buildSrc = () => buildProject("src");
 
 task("build-src", series(preSrc, buildSrc));
 
+/**
+ * @param {string} entrypoint
+ * @param {string} output
+ */
+async function runDtsBundler(entrypoint, output) {
+    // Want to preserve @internal? Pass `--stripInternal=false` when running the dts task.
+    await exec(process.execPath, [
+        "./scripts/dtsBundler.js",
+        "--entrypoint",
+        entrypoint,
+        "--output",
+        output,
+        `--stripInternal=${cmdLineOptions.stripInternal}`,
+    ]);
+}
+
 /** @type {string | undefined} */
 let copyrightHeader;
 function getCopyrightHeader() {
@@ -268,6 +284,7 @@ const esbuildServices = esbuildTask("./src/typescript/typescript.ts", "./built/l
 const writeServicesCJSShim = () => writeCJSReexport("./built/local/typescript/typescript.js", "./built/local/typescript.js");
 const buildServicesProject = () => buildProject("src/typescript");
 
+// TODO(jakebailey): rename this; no longer "services".
 const buildServices = () => {
     if (cmdLineOptions.bundle) return esbuildServices.build();
     writeServicesCJSShim();
@@ -297,6 +314,9 @@ task("watch-services").flags = {
     "   --built": "Compile using the built version of the compiler."
 };
 
+const dtsServices = () => runDtsBundler("./built/local/typescript/typescript.d.ts", "./built/local/typescript.d.ts");
+task("dts-services", series(preBuild, buildServicesProject, dtsServices));
+task("dts-services").description = "Builds typescript.d.ts";
 
 const esbuildServer = esbuildTask("./src/tsserver/server.ts", "./built/local/tsserver.js", /* exportIsTsObject */ true);
 const writeServerCJSShim = () => writeCJSReexport("./built/local/tsserver/server.js", "./built/local/tsserver.js");
@@ -348,10 +368,11 @@ task("watch-min").flags = {
 const esbuildLssl = esbuildTask("./src/tsserverlibrary/tsserverlibrary.ts", "./built/local/tsserverlibrary.js", /* exportIsTsObject */ true);
 const writeLsslCJSShim = () => writeCJSReexport("./built/local/tsserverlibrary/tsserverlibrary.js", "./built/local/tsserverlibrary.js");
 
+const buildLsslProject = () => buildProject("src/tsserverlibrary");
 const buildLssl = () => {
     if (cmdLineOptions.bundle) return esbuildLssl.build();
     writeLsslCJSShim();
-    return buildProject("src/tsserverlibrary");
+    return buildLsslProject();
 };
 task("lssl", series(preBuild, buildLssl));
 task("lssl").description = "Builds language service server library";
@@ -375,6 +396,14 @@ task("watch-lssl").flags = {
     "   --built": "Compile using the built version of the compiler."
 };
 
+const dtsLssl = () => runDtsBundler("./built/local/tsserverlibrary/tsserverlibrary.d.ts", "./built/local/tsserverlibrary.d.ts");
+task("dts-lssl", series(preBuild, buildLsslProject, dtsLssl));
+task("dts-lssl").description = "Builds tsserverlibrary.d.ts";
+
+// TODO(jakebailey): this is probably not efficient, but, gulp.
+const dts = series(preBuild, parallel(buildServicesProject, buildLsslProject), parallel(dtsServices, dtsLssl));
+task("dts", dts);
+
 const testRunner = "./built/local/run.js";
 const esbuildTests = esbuildTask("./src/testRunner/_namespaces/Harness.ts", testRunner);
 const writeTestsCJSShim = () => writeCJSReexport("./built/local/testRunner/runner.js", testRunner);
@@ -488,7 +517,7 @@ const buildOtherOutputs = parallel(buildCancellationToken, buildTypingsInstaller
 task("other-outputs", series(preBuild, buildOtherOutputs));
 task("other-outputs").description = "Builds miscelaneous scripts and documents distributed with the LKG";
 
-task("local", series(preBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs)));
+task("local", series(preBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs, dts)));
 task("local").description = "Builds the full compiler and services";
 task("local").flags = {
     "   --built": "Compile using the built version of the compiler."
@@ -504,7 +533,7 @@ const preTest = parallel(buildTsc, buildTests, buildServices, buildLssl);
 preTest.displayName = "preTest";
 
 const runTests = () => runConsoleTests(testRunner, "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
-task("runtests", series(preBuild, preTest, runTests));
+task("runtests", series(preBuild, preTest, dts, runTests));
 task("runtests").description = "Runs the tests using the built run.js file.";
 task("runtests").flags = {
     "-t --tests=<regex>": "Pattern for tests to run.",
@@ -523,7 +552,7 @@ task("runtests").flags = {
 };
 
 const runTestsParallel = () => runConsoleTests(testRunner, "min", /*runInParallel*/ cmdLineOptions.workers > 1, /*watchMode*/ false);
-task("runtests-parallel", series(preBuild, preTest, runTestsParallel));
+task("runtests-parallel", series(preBuild, preTest, dts, runTestsParallel));
 task("runtests-parallel").description = "Runs all the tests in parallel using the built run.js file.";
 task("runtests-parallel").flags = {
     "   --light": "Run tests in light mode (fewer verifications, but tests run faster).",
@@ -613,8 +642,7 @@ const produceLKG = async () => {
     }
 };
 
-// TODO(jakebailey): dependencies on dts
-task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs), produceLKG));
+task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs, dts), produceLKG));
 task("LKG").description = "Makes a new LKG out of the built js files";
 task("LKG").flags = {
     "   --built": "Compile using the built version of the compiler.",

scripts/build/options.js

@@ -6,7 +6,7 @@ const ci = ["1", "true"].includes(process.env.CI);
 
 /** @type {CommandLineOptions} */
 module.exports = minimist(process.argv.slice(2), {
-    boolean: ["dirty", "light", "colors", "lkg", "soft", "fix", "failed", "keepFailed", "force", "built", "ci", "bundle"],
+    boolean: ["dirty", "light", "colors", "lkg", "soft", "fix", "failed", "keepFailed", "force", "built", "ci", "bundle", "stripInternal"],
     string: ["browser", "tests", "break", "host", "reporter", "stackTraceLimit", "timeout", "shards", "shardId"],
     alias: {
         /* eslint-disable quote-props */
@@ -41,7 +41,8 @@ module.exports = minimist(process.argv.slice(2), {
         dirty: false,
         built: false,
         ci,
-        bundle: true
+        bundle: true,
+        stripInternal: true,
     }
 });
 
@@ -74,5 +75,6 @@ if (module.exports.built) {
  * @property {string} shards
  * @property {string} shardId
  * @property {string} break
+ * @property {boolean} stripInternal
  */
 void 0;

scripts/dtsBundler.ts

@@ -0,0 +1,219 @@
+/**
+ * WARNING: this is a very, very rudimentary d.ts bundler; it only works
+ * in the TS project thanks to our history using namespaces, which has
+ * prevented us from duplicating names across files, and allows us to
+ * bundle as namespaces again, even though the project is modules.
+ */
+
+import * as assert from "assert";
+import * as fs from "fs";
+import * as path from "path";
+import * as minimist from "minimist";
+import * as ts from "../lib/typescript";
+
+const options = minimist(process.argv.slice(2), {
+    boolean: ["stripInternal"],
+    string: ["project", "entrypoint", "output"],
+    default: {
+        stripInternal: true,
+    },
+});
+
+const stripInternal = !!options.stripInternal;
+const entrypoint = options.entrypoint;
+const output = options.output;
+
+
+assert(typeof entrypoint === "string" && entrypoint);
+assert(typeof output === "string" && output);
+
+console.log(`Bundling ${entrypoint} to ${output}`);
+
+const newLineKind = ts.NewLineKind.LineFeed;
+const newLine = newLineKind === ts.NewLineKind.LineFeed ? "\n" : "\r\n";
+
+function isDeclarationStatement(node: ts.Node): node is ts.DeclarationStatement {
+    return (ts as any).isDeclarationStatement(node);
+}
+
+function isInternalDeclaration(node: ts.Node): boolean {
+    return (ts as any).isInternalDeclaration(node, node.getSourceFile());
+}
+
+function getParentVariableStatement(node: ts.VariableDeclaration): ts.VariableStatement {
+    const declarationList = node.parent as ts.VariableDeclarationList;
+    assert(ts.isVariableDeclarationList(declarationList), `expected VariableDeclarationList at ${nodeToLocation(node)}`);
+    assert(declarationList.declarations.length === 1, `expected VariableDeclarationList of length 1 at ${nodeToLocation(node)}`);
+    const variableStatement = declarationList.parent;
+    assert(ts.isVariableStatement(variableStatement), `expected VariableStatement at ${nodeToLocation(node)}`);
+    return variableStatement;
+}
+
+function getDeclarationStatement(node: ts.Declaration): ts.Statement | undefined {
+    if (ts.isVariableDeclaration(node)) {
+        return getParentVariableStatement(node);
+    }
+    else if (isDeclarationStatement(node)) {
+        return node;
+    }
+    return undefined;
+}
+
+const nullTransformationContext: ts.TransformationContext = (ts as any).nullTransformationContext;
+
+const program = ts.createProgram([entrypoint], { target: ts.ScriptTarget.ES5 });
+
+const typeChecker = program.getTypeChecker();
+
+const sourceFile = program.getSourceFile(entrypoint);
+assert(sourceFile);
+const moduleSymbol = typeChecker.getSymbolAtLocation(sourceFile);
+assert(moduleSymbol);
+
+const printer = ts.createPrinter({ newLine: newLineKind });
+
+
+const lines: string[] = [];
+const indent = "    ";
+let currentIndent = "";
+
+function increaseIndent() {
+    currentIndent += indent;
+}
+
+function decreaseIndent() {
+    currentIndent = currentIndent.slice(indent.length);
+}
+
+function write(s: string) {
+    if (!s) {
+        lines.push("");
+    }
+    else {
+        lines.push(...s.split(/\r?\n/).filter(line => line).map(line => (currentIndent + line).trimEnd()));
+    }
+}
+
+const containsPublicAPICache = new Map<ts.Symbol, boolean>();
+
+function containsPublicAPI(symbol: ts.Symbol): boolean {
+    const cached = containsPublicAPICache.get(symbol);
+    if (cached !== undefined) {
+        return cached;
+    }
+
+    const result = containsPublicAPIWorker();
+    containsPublicAPICache.set(symbol, result);
+    return result;
+
+    function containsPublicAPIWorker(): boolean {
+        if (!symbol.declarations?.length) {
+            return false;
+        }
+
+        if (symbol.flags & ts.SymbolFlags.Alias) {
+            const resolved = typeChecker.getAliasedSymbol(symbol);
+            return containsPublicAPI(resolved);
+        }
+
+        if (symbol.flags & ts.SymbolFlags.ValueModule) {
+            for (const me of typeChecker.getExportsOfModule(symbol)) {
+                if (containsPublicAPI(me)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        for (const decl of symbol.declarations) {
+            const statement = getDeclarationStatement(decl);
+            if (statement && !isInternalDeclaration(statement)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}
+
+function nodeToLocation(decl: ts.Node): string {
+    const sourceFile = decl.getSourceFile();
+    const lc = sourceFile.getLineAndCharacterOfPosition(decl.pos);
+    return `${sourceFile.fileName}:${lc.line}:${lc.character}`;
+}
+
+function emitAsNamespace(name: string, moduleSymbol: ts.Symbol) {
+    assert(moduleSymbol.flags & ts.SymbolFlags.ValueModule);
+
+    if (name === "ts") {
+        // We will write `export = ts` at the end.
+        write(`declare namespace ${name} {`);
+    }
+    else {
+        // No export modifier; we are already in the namespace.
+        write(`namespace ${name} {`);
+    }
+    increaseIndent();
+
+    const moduleExports = typeChecker.getExportsOfModule(moduleSymbol);
+    for (const me of moduleExports) {
+        if (stripInternal && !containsPublicAPI(me)) {
+            continue;
+        }
+
+        assert(me.declarations?.length);
+
+        if (me.flags & ts.SymbolFlags.Alias) {
+            const resolved = typeChecker.getAliasedSymbol(me);
+            emitAsNamespace(me.name, resolved);
+            continue;
+        }
+
+        for (const decl of me.declarations) {
+            let statement = getDeclarationStatement(decl);
+
+            if (!statement) {
+                throw new Error(`Unhandled declaration for ${me.name} at ${nodeToLocation(decl)}`);
+            }
+
+            // Ignore an internal declaration.
+            if (stripInternal && isInternalDeclaration(statement)) {
+                continue;
+            }
+
+            // Remove internal components and declare/const keywords.
+            statement = ts.visitEachChild(statement, (node) => {
+                if (stripInternal && isInternalDeclaration(node)) {
+                    return undefined;
+                }
+
+                switch (node.kind) {
+                    case ts.SyntaxKind.DeclareKeyword: // No need to emit this in d.ts files.
+                    case ts.SyntaxKind.ConstKeyword:   // Remove const from const enums.
+                    case ts.SyntaxKind.ExportKeyword:  // No export modifier; we are already in the namespace.
+                        return undefined;
+                }
+
+                return node;
+            }, nullTransformationContext);
+
+            write(printer.printNode(ts.EmitHint.Unspecified, statement, decl.getSourceFile()));
+        }
+    }
+
+    decreaseIndent();
+    write(`}`);
+}
+
+emitAsNamespace("ts", moduleSymbol);
+
+write("export = ts;");
+
+const copyrightNotice = fs.readFileSync(path.join(__dirname, "..", "CopyrightNotice.txt"), "utf-8");
+const outputContents = copyrightNotice + lines.join(newLine);
+
+if (stripInternal && outputContents.includes("@internal")) {
+    console.error("Output includes untrimmed @internal nodes!");
+}
+
+fs.writeFileSync(output, outputContents);

scripts/produceLKG.ts

@@ -7,7 +7,6 @@ import * as glob from "glob";
 const root = path.join(__dirname, "..");
 const source = path.join(root, "built/local");
 const dest = path.join(root, "lib");
-const copyright = fs.readFileSync(path.join(__dirname, "../CopyrightNotice.txt"), "utf-8");
 
 async function produceLKG() {
     console.log(`Building LKG from ${source} to ${dest}`);
@@ -74,19 +73,14 @@ async function copyScriptOutputs() {
 }
 
 async function copyDeclarationOutputs() {
-    await copyWithCopyright("tsserverlibrary.d.ts");
-    await copyWithCopyright("typescript.d.ts");
+    await copyFromBuiltLocal("tsserverlibrary.d.ts");
+    await copyFromBuiltLocal("typescript.d.ts");
 }
 
 async function writeGitAttributes() {
     await fs.writeFile(path.join(dest, ".gitattributes"), `* text eol=lf`, "utf-8");
 }
 
-async function copyWithCopyright(fileName: string, destName = fileName) {
-    const content = await fs.readFile(path.join(source, fileName), "utf-8");
-    await fs.writeFile(path.join(dest, destName), copyright + "\n" + content);
-}
-
 async function copyFromBuiltLocal(fileName: string) {
     await fs.copy(path.join(source, fileName), path.join(dest, fileName));
 }