Add dts bundling

Author: jakebailey

.gitignore

@@ -47,6 +47,7 @@ scripts/configurePrerelease.js
 scripts/configureLanguageServiceBuild.js
 scripts/open-user-pr.js
 scripts/open-cherry-pick-pr.js
+scripts/dtsBundler.js
 scripts/processDiagnosticMessages.d.ts
 scripts/processDiagnosticMessages.js
 scripts/produceLKG.js

Gulpfile.js

@@ -127,6 +127,15 @@ const buildSrc = () => buildProject("src");
 // But, if we are bundling, we are running only d.ts emit, so maybe this is fast?
 task("build-src", series(preSrc, buildSrc));
 
+/**
+ * @param {string} projectPath
+ * @param {string} entrypoint
+ * @param {string} output
+ */
+async function runDtsBundler(projectPath, entrypoint, output) {
+    await exec(process.execPath, ["./scripts/dtsBundler.js", projectPath, entrypoint, output]);
+}
+
 /** @type {string | undefined} */
 let copyrightHeader;
 function getCopyrightHeader() {
@@ -143,6 +152,7 @@ function getCopyrightHeader() {
  * @param {boolean} performanceMatters True if this is a bundle where performance matters, so should be optimized at the cost of build time.
  */
 function esbuildTask(entrypoint, outfile, exportIsTsObject = false, performanceMatters = false) {
+    performanceMatters = false;
     const preBabel = `${outfile}.tmp.js`;
 
     /** @type {esbuild.BuildOptions} */
@@ -251,10 +261,13 @@ const preBuild = cmdLineOptions.lkg ? lkgPreBuild : localPreBuild;
 const esbuildServices = esbuildTask("./src/typescript/typescript.ts", "./built/local/typescript.js", /* exportIsTsObject */ true, /* performanceMatters */ true);
 
 // TODO(jakebailey): rename this; no longer "services".
+
+const buildServicesProject = () => buildProject("src/typescript");
+
 const buildServices = () => {
     if (cmdLineOptions.bundle) return esbuildServices.build();
     writeCJSReexport("./built/local/typescript/typescript.js", "./built/local/typescript.js");
-    return buildProject("src/typescript");
+    return buildServicesProject();
 };
 
 task("services", series(preBuild, buildServices));
@@ -277,6 +290,9 @@ task("watch-services").flags = {
     "   --built": "Compile using the built version of the compiler."
 };
 
+const dtsServices = () => runDtsBundler("./src/typescript/tsconfig.json", "./built/local/typescript/typescript.d.ts", "./built/local/typescript.d.ts");
+task("dts-services", series(preBuild, buildServicesProject, dtsServices));
+task("dts-services").description = "Builds typescript.d.ts";
 
 const esbuildServer = esbuildTask("./src/tsserver/server.ts", "./built/local/tsserver.js", /* exportIsTsObject */ true, /* performanceMatters */ true);
 
@@ -323,10 +339,11 @@ task("watch-min").flags = {
 
 const esbuildLssl = esbuildTask("./src/tsserverlibrary/tsserverlibrary.ts", "./built/local/tsserverlibrary.js", /* exportIsTsObject */ true, /* performanceMatters */ true);
 
+const buildLsslProject = () => buildProject("src/tsserverlibrary");
 const buildLssl = () => {
     if (cmdLineOptions.bundle) return esbuildLssl.build();
     writeCJSReexport("./built/local/tsserverlibrary/tsserverlibrary.js", "./built/local/tsserverlibrary.js");
-    return buildProject("src/tsserverlibrary");
+    return buildLsslProject();
 };
 task("lssl", series(preBuild, buildLssl));
 task("lssl").description = "Builds language service server library";
@@ -348,6 +365,14 @@ task("watch-lssl").flags = {
     "   --built": "Compile using the built version of the compiler."
 };
 
+const dtsLssl = () => runDtsBundler("./src/tsserverlibrary/tsconfig.json", "./built/local/tsserverlibrary/tsserverlibrary.d.ts", "./built/local/tsserverlibrary.d.ts");
+task("dts-lssl", series(preBuild, buildLsslProject, dtsLssl));
+task("dts-lssl").description = "Builds tsserverlibrary.d.ts";
+
+// TODO(jakebailey): this is probably not efficient, but, gulp.
+const dts = series(preBuild, parallel(dtsServices, dtsLssl));
+task("dts", dts);
+
 const testRunner = "./built/local/run.js";
 const esbuildTests = esbuildTask("./src/testRunner/_namespaces/Harness.ts", testRunner);
 
@@ -458,7 +483,7 @@ const buildOtherOutputs = parallel(buildCancellationToken, buildTypingsInstaller
 task("other-outputs", series(preBuild, buildOtherOutputs));
 task("other-outputs").description = "Builds miscelaneous scripts and documents distributed with the LKG";
 
-task("local", series(preBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs)));
+task("local", series(preBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs, dts)));
 task("local").description = "Builds the full compiler and services";
 task("local").flags = {
     "   --built": "Compile using the built version of the compiler."
@@ -474,7 +499,7 @@ const preTest = parallel(buildTsc, buildTests, buildServices, buildLssl);
 preTest.displayName = "preTest";
 
 const runTests = () => runConsoleTests(testRunner, "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
-task("runtests", series(preBuild, preTest, runTests));
+task("runtests", series(preBuild, preTest, dts, runTests));
 task("runtests").description = "Runs the tests using the built run.js file.";
 task("runtests").flags = {
     "-t --tests=<regex>": "Pattern for tests to run.",
@@ -493,7 +518,7 @@ task("runtests").flags = {
 };
 
 const runTestsParallel = () => runConsoleTests(testRunner, "min", /*runInParallel*/ cmdLineOptions.workers > 1, /*watchMode*/ false);
-task("runtests-parallel", series(preBuild, preTest, runTestsParallel));
+task("runtests-parallel", series(preBuild, preTest, dts, runTestsParallel));
 task("runtests-parallel").description = "Runs all the tests in parallel using the built run.js file.";
 task("runtests-parallel").flags = {
     "   --light": "Run tests in light mode (fewer verifications, but tests run faster).",
@@ -597,8 +622,7 @@ const produceLKG = async () => {
     }
 };
 
-// TODO(jakebailey): dependencies on dts
-task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs), produceLKG));
+task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs, dts), produceLKG));
 task("LKG").description = "Makes a new LKG out of the built js files";
 task("LKG").flags = {
     "   --built": "Compile using the built version of the compiler.",

scripts/dtsBundler.ts

@@ -0,0 +1,210 @@
+/**
+ * WARNING: this is a very, very rudimentary d.ts bundler; it only works
+ * in the TS project thanks to our history using namespaces, which has
+ * prevented us from duplicating names across files, and allows us to
+ * bundle as namespaces again, even though the project is modules.
+ */
+
+import * as assert from "assert";
+import * as fs from "fs";
+import * as path from "path";
+import * as ts from "../lib/typescript";
+
+const [
+    projectPath,
+    entrypoint,
+    output,
+] = process.argv.slice(2);
+
+assert(projectPath);
+assert(entrypoint);
+assert(output);
+
+console.log(`Bundling ${entrypoint} to ${output}, using project ${projectPath}`);
+
+const newLineKind = ts.NewLineKind.LineFeed;
+const newLine = newLineKind === ts.NewLineKind.LineFeed ? "\n" : "\r\n";
+
+function isDeclarationStatement(node: ts.Node): node is ts.DeclarationStatement {
+    return (ts as any).isDeclarationStatement(node);
+}
+
+function isInternalDeclaration(node: ts.Node): boolean {
+    return (ts as any).isInternalDeclaration(node, node.getSourceFile());
+}
+
+function getParentVariableStatement(node: ts.VariableDeclaration): ts.VariableStatement {
+    const declarationList = node.parent as ts.VariableDeclarationList;
+    assert(ts.isVariableDeclarationList(declarationList), `expected VariableDeclarationList at ${nodeToLocation(node)}`);
+    assert(declarationList.declarations.length === 1, `expected VariableDeclarationList of length 1 at ${nodeToLocation(node)}`);
+    const variableStatement = declarationList.parent;
+    assert(ts.isVariableStatement(variableStatement), `expected VariableStatement at ${nodeToLocation(node)}`);
+    return variableStatement;
+}
+
+function getDeclarationStatement(node: ts.Declaration): ts.Statement | undefined {
+    if (ts.isVariableDeclaration(node)) {
+        return getParentVariableStatement(node);
+    }
+    else if (isDeclarationStatement(node)) {
+        return node;
+    }
+    return undefined;
+}
+
+const nullTransformationContext: ts.TransformationContext = (ts as any).nullTransformationContext;
+
+const tsconfigContents = ts.readJsonConfigFile(path.join(projectPath, "tsconfig.json"), ts.sys.readFile);
+const tsconfig = ts.parseJsonConfigFileContent(tsconfigContents, ts.sys, projectPath);
+const program = ts.createProgram([entrypoint], tsconfig.options);
+
+const typeChecker = program.getTypeChecker();
+
+const sourceFile = program.getSourceFile(entrypoint);
+assert(sourceFile);
+const moduleSymbol = typeChecker.getSymbolAtLocation(sourceFile);
+assert(moduleSymbol);
+
+const printer = ts.createPrinter({ newLine: newLineKind });
+
+
+const lines: string[] = [];
+const indent = "    ";
+let currentIndent = "";
+
+function increaseIndent() {
+    currentIndent += indent;
+}
+
+function decreaseIndent() {
+    currentIndent = currentIndent.slice(indent.length);
+}
+
+function write(s: string) {
+    if (!s) {
+        lines.push("");
+    }
+    else {
+        lines.push(...s.split(/\r?\n/).filter(line => line).map(line => (currentIndent + line).trimEnd()));
+    }
+}
+
+const containsPublicAPICache = new Map<ts.Symbol, boolean>();
+
+function containsPublicAPI(symbol: ts.Symbol): boolean {
+    const cached = containsPublicAPICache.get(symbol);
+    if (cached !== undefined) {
+        return cached;
+    }
+
+    const result = containsPublicAPIWorker();
+    containsPublicAPICache.set(symbol, result);
+    return result;
+
+    function containsPublicAPIWorker(): boolean {
+        if (!symbol.declarations?.length) {
+            return false;
+        }
+
+        if (symbol.flags & ts.SymbolFlags.Alias) {
+            const resolved = typeChecker.getAliasedSymbol(symbol);
+            return containsPublicAPI(resolved);
+        }
+
+        if (symbol.flags & ts.SymbolFlags.ValueModule) {
+            for (const me of typeChecker.getExportsOfModule(symbol)) {
+                if (containsPublicAPI(me)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        for (const decl of symbol.declarations) {
+            const statement = getDeclarationStatement(decl);
+            if (statement && !isInternalDeclaration(statement)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}
+
+function nodeToLocation(decl: ts.Node): string {
+    const sourceFile = decl.getSourceFile();
+    const lc = sourceFile.getLineAndCharacterOfPosition(decl.pos);
+    return `${sourceFile.fileName}:${lc.line}:${lc.character}`;
+}
+
+function emitAsNamespace(name: string, moduleSymbol: ts.Symbol) {
+    assert(moduleSymbol.flags & ts.SymbolFlags.ValueModule);
+
+    if (name === "ts") {
+        // We will write `export = ts` at the end.
+        write(`declare namespace ${name} {`);
+    }
+    else {
+        write(`export namespace ${name} {`);
+    }
+    increaseIndent();
+
+    const moduleExports = typeChecker.getExportsOfModule(moduleSymbol);
+    for (const me of moduleExports) {
+        if (!containsPublicAPI(me)) {
+            continue;
+        }
+
+        assert(me.declarations?.length);
+
+        if (me.flags & ts.SymbolFlags.Alias) {
+            const resolved = typeChecker.getAliasedSymbol(me);
+            emitAsNamespace(me.name, resolved);
+            continue;
+        }
+
+        for (const decl of me.declarations) {
+            let statement = getDeclarationStatement(decl);
+
+            if (!statement) {
+                throw new Error(`Unhandled declaration for ${me.name} at ${nodeToLocation(decl)}`);
+            }
+
+            // Ignore an internal declaration.
+            if (isInternalDeclaration(statement)) {
+                continue;
+            }
+
+            // Remove internal components and declare/const keywords.
+            statement = ts.visitEachChild(statement, (node) => {
+                if (isInternalDeclaration(node)) {
+                    return undefined;
+                }
+
+                if (node.kind === ts.SyntaxKind.DeclareKeyword || node.kind === ts.SyntaxKind.ConstKeyword) {
+                    return undefined;
+                }
+
+                return node;
+            }, nullTransformationContext);
+
+            write(printer.printNode(ts.EmitHint.Unspecified, statement, decl.getSourceFile()));
+        }
+    }
+
+    decreaseIndent();
+    write(`}`);
+}
+
+emitAsNamespace("ts", moduleSymbol);
+
+write("export = ts;");
+
+const copyrightNotice = fs.readFileSync(path.join(__dirname, "..", "CopyrightNotice.txt"), "utf-8");
+const outputContents = copyrightNotice + lines.join(newLine);
+
+if (outputContents.includes("@internal")) {
+    console.error("Output includes untrimmed @internal nodes!");
+}
+
+fs.writeFileSync(output, outputContents);

src/compiler/commandLineParser.ts

@@ -2081,7 +2081,7 @@ function getTsconfigRootOptionsMap() {
 }
 
 /** @internal */
-interface JsonConversionNotifier {
+export interface JsonConversionNotifier {
     /**
      * Notifies parent option object is being set with the optionKey and a valid optionValue
      * Currently it notifies only if there is element with type object (parentOption) and

src/compiler/factory/utilities.ts

@@ -2,11 +2,12 @@ import {
     AccessorDeclaration, addEmitFlags, AdditiveOperator, AdditiveOperatorOrHigher, AssertionLevel,
     AssignmentOperatorOrHigher, BinaryExpression, BinaryOperator, BinaryOperatorToken, BindingOrAssignmentElement,
     BindingOrAssignmentElementRestIndicator, BindingOrAssignmentElementTarget, BindingOrAssignmentPattern,
-    BitwiseOperator, BitwiseOperatorOrHigher, BooleanLiteral, CharacterCodes, CommaListExpression,
+    BitwiseOperator, BitwiseOperatorOrHigher, Block, BooleanLiteral, CharacterCodes, CommaListExpression,
     compareStringsCaseSensitive, CompilerOptions, Debug, Declaration, EmitFlags, EmitHelperFactory, EmitHost,
     EmitResolver, EntityName, EqualityOperator, EqualityOperatorOrHigher, ExclamationToken, ExponentiationOperator,
     ExportDeclaration, Expression, ExpressionStatement, externalHelpersModuleNameText, first, firstOrUndefined,
     ForInitializer, GeneratedIdentifier, GeneratedIdentifierFlags, GeneratedNamePart, GeneratedPrivateIdentifier,
+    GetAccessorDeclaration,
     getAllAccessorDeclarations, getEmitFlags, getEmitHelpers, getEmitModuleKind, getESModuleInterop,
     getExternalModuleName, getExternalModuleNameFromPath, getJSDocType, getJSDocTypeTag, getModifiers,
     getNamespaceDeclarationNode, getOrCreateEmitNode, getOriginalNode, getParseTreeNode,
@@ -26,7 +27,7 @@ import {
     NumericLiteral, ObjectLiteralElementLike, ObjectLiteralExpression, or, OuterExpression, OuterExpressionKinds,
     outFile, parseNodeFactory, PlusToken, PostfixUnaryExpression, PrefixUnaryExpression, PrivateIdentifier,
     PropertyAssignment, PropertyDeclaration, PropertyName, pushIfUnique, QuestionToken, ReadonlyKeyword,
-    RelationalOperator, RelationalOperatorOrHigher, setOriginalNode, setParent, setStartsOnNewLine, setTextRange,
+    RelationalOperator, RelationalOperatorOrHigher, SetAccessorDeclaration, setOriginalNode, setParent, setStartsOnNewLine, setTextRange,
     ShiftOperator, ShiftOperatorOrHigher, ShorthandPropertyAssignment, some, SourceFile, Statement, StringLiteral,
     SyntaxKind, TextRange, ThisTypeNode, Token, TypeNode, TypeParameterDeclaration,
 } from "../_namespaces/ts";
@@ -185,7 +186,7 @@ export function createForOfBindingStatement(factory: NodeFactory, node: ForIniti
 }
 
 /** @internal */
-export function insertLeadingStatement(factory: NodeFactory, dest: Statement, source: Statement) {
+export function insertLeadingStatement(factory: NodeFactory, dest: Statement, source: Statement): Block {
     if (isBlock(dest)) {
         return factory.updateBlock(dest, setTextRange(factory.createNodeArray([source, ...dest.statements]), dest.statements));
     }
@@ -1456,7 +1457,7 @@ export function createAccessorPropertyBackingField(factory: NodeFactory, node: P
  *
  * @internal
  */
-export function createAccessorPropertyGetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName) {
+export function createAccessorPropertyGetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName): GetAccessorDeclaration {
     return factory.createGetAccessorDeclaration(
         modifiers,
         name,
@@ -1478,7 +1479,7 @@ export function createAccessorPropertyGetRedirector(factory: NodeFactory, node:
  *
  * @internal
  */
-export function createAccessorPropertySetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName) {
+export function createAccessorPropertySetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName): SetAccessorDeclaration {
     return factory.createSetAccessorDeclaration(
         modifiers,
         name,