[6.0] Support macros when cross-compiling (#7640)

**Explanation**: Macros cross-compiled by SwiftPM with Swift SDKs should
be correctly built, loaded, and evaluated for the host triple.
**Scope**: isolated to modules dependency resolution and llbuild, does
not impact code related to XCBuild.
**Risk**: medium, known issues were addressed on `main` and are
cherry-picked here, with no new issues reported for a few weeks now.
**Testing**: added unit tests, manual end-to-end testing done with
existing Swift SDKs.
**Issue**: rdar://105991372
**Reviewers**: @bnbarham @xedin @neonichu

(cherry picked from commit cb3b085,
#7353)

```
# Conflicts:
#	CHANGELOG.md
#	Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift
#	Sources/Build/BuildManifest/LLBuildManifestBuilder.swift
#	Sources/Build/BuildPlan/BuildPlan.swift
#	Sources/Commands/SwiftTestCommand.swift
#	Sources/Commands/Utilities/PluginDelegate.swift
#	Sources/Commands/Utilities/TestingSupport.swift
#	Sources/PackageGraph/ModulesGraph+Loading.swift
#	Sources/PackageGraph/ModulesGraph.swift
#	Sources/SPMTestSupport/MockBuildTestHelper.swift
#	Sources/SPMTestSupport/MockPackageGraphs.swift
#	Sources/SPMTestSupport/PackageGraphTester.swift
#	Sources/SPMTestSupport/ResolvedTarget+Mock.swift
#	Tests/BuildTests/ClangTargetBuildDescriptionTests.swift
#	Tests/BuildTests/CrossCompilationBuildPlanTests.swift
#	Tests/SourceKitLSPAPITests/SourceKitLSPAPITests.swift
```

(cherry picked from commit b9eb3c1,
#7493)

```
# Conflicts:
#	Sources/PackageGraph/ModulesGraph+Loading.swift
#	Sources/PackageGraph/Resolution/ResolvedPackage.swift
```
(cherry picked from commit 5a4c024,
#7508)

```
# Conflicts:
#	Sources/Commands/SwiftBuildCommand.swift
#	Sources/Commands/SwiftTestCommand.swift
#	Sources/Commands/Utilities/TestingSupport.swift
```

(cherry picked from commit 8db2401,
#7519)

```
# Conflicts:
#	Tests/BuildTests/CrossCompilationBuildPlanTests.swift
```

---------

Co-authored-by: Jonathan Grynspan <[email protected]>
Co-authored-by: Ben Barham <[email protected]>
Co-authored-by: Yuta Saito <[email protected]>
Co-authored-by: Pavel Yaskevich <[email protected]>

Author: 5 people

CHANGELOG.md

@@ -27,13 +27,18 @@ Swift 6.0
 
   `// swift-tools-version:` can now be specified on subsequent lines of `Package.swift`, for example when first few lines are required to contain a licensing comment header.
 
+* [#7118]
+
+  Macros cross-compiled by SwiftPM with Swift SDKs are now correctly built, loaded, and evaluated for the host triple.
+
 Swift 5.10
 -----------
+
 * [#7010]
 
   On macOS, `swift build` and `swift run` now produce binaries that allow backtraces in debug builds. Pass `SWIFT_BACKTRACE=enable=yes` environment variable to enable backtraces on such binaries when running them.
 
-* [7101]
+* [#7101]
 
    Binary artifacts are now cached along side repository checkouts so they do not need to be re-downloaded across projects.
 

Sources/Basics/Collections/IdentifiableSet.swift

@@ -47,13 +47,22 @@ public struct IdentifiableSet<Element: Identifiable>: Collection {
     }
 
     public subscript(id: Element.ID) -> Element? {
-        self.storage[id]
+        get {
+            self.storage[id]
+        }
+        set {
+            self.storage[id] = newValue
+        }
     }
 
     public func index(after i: Index) -> Index {
         Index(storageIndex: self.storage.elements.index(after: i.storageIndex))
     }
 
+    public mutating func insert(_ element: Element) {
+        self.storage[element.id] = element
+    }
+
     public func union(_ otherSequence: some Sequence<Element>) -> Self {
         var result = self
         for element in otherSequence {

Sources/Basics/Errors.swift

@@ -21,8 +21,7 @@ public struct InternalError: Error {
     private let description: String
     public init(_ description: String) {
         assertionFailure(description)
-        self
-            .description =
+        self.description =
             "Internal error. Please file a bug at https://github.com/apple/swift-package-manager/issues with this info. \(description)"
     }
 }

Sources/Build/BuildDescription/ClangTargetBuildDescription.swift

@@ -53,7 +53,7 @@ public final class ClangTargetBuildDescription {
 
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
-        guard !resources.isEmpty else {
+        guard !self.resources.isEmpty else {
             return .none
         }
 
@@ -133,7 +133,7 @@ public final class ClangTargetBuildDescription {
         self.target = target
         self.toolsVersion = toolsVersion
         self.buildParameters = buildParameters
-        self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
+        self.tempsPath = target.tempsPath(buildParameters)
         self.derivedSources = Sources(paths: [], root: tempsPath.appending("DerivedSources"))
 
         // We did not use to apply package plugins to C-family targets in prior tools-versions, this preserves the behavior.
@@ -225,7 +225,7 @@ public final class ClangTargetBuildDescription {
         if self.buildParameters.triple.isDarwin() {
             args += ["-fobjc-arc"]
         }
-        args += try buildParameters.targetTripleArgs(for: target)
+        args += try self.buildParameters.tripleArgs(for: target)
 
         args += optimizationArguments
         args += activeCompilationConditions

Sources/Build/BuildDescription/PluginDescription.swift

@@ -29,6 +29,9 @@ public final class PluginDescription: Codable {
     /// the plugin).
     public let targetName: String
 
+    /// The language-level target name.
+    public let targetC99Name: String
+
     /// The names of any plugin products in that package that vend the plugin
     /// to other packages.
     public let productNames: [String]
@@ -56,6 +59,7 @@ public final class PluginDescription: Codable {
 
         self.package = package.identity
         self.targetName = target.name
+        self.targetC99Name = target.c99name
         self.productNames = products.map(\.name)
         self.toolsVersion = toolsVersion
         self.sources = target.sources

Sources/Build/BuildDescription/ProductBuildDescription.swift

@@ -322,7 +322,7 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
         // setting is the package-level right now. We might need to figure out a better
         // answer for libraries if/when we support specifying deployment target at the
         // target-level.
-        args += try self.buildParameters.targetTripleArgs(for: self.product.targets[self.product.targets.startIndex])
+        args += try self.buildParameters.tripleArgs(for: self.product.targets[self.product.targets.startIndex])
 
         // Add arguments from declared build settings.
         args += self.buildSettingsFlags
@@ -357,7 +357,7 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
         // Library search path for the toolchain's copy of SwiftSyntax.
         #if BUILD_MACROS_AS_DYLIBS
         if product.type == .macro {
-            args += try ["-L", buildParameters.toolchain.hostLibDir.pathString]
+            args += try ["-L", defaultBuildParameters.toolchain.hostLibDir.pathString]
         }
         #endif
 

Sources/Build/BuildDescription/ResolvedModule+BuildDescription.swift

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2015-2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import struct Basics.AbsolutePath
+import struct PackageGraph.ResolvedModule
+
+import SPMBuildCore
+
+extension ResolvedModule {
+    func tempsPath(_ buildParameters: BuildParameters) -> AbsolutePath {
+        let suffix = buildParameters.suffix
+        return buildParameters.buildPath.appending(component: "\(self.c99name)\(suffix).build")
+    }
+}

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -43,7 +43,7 @@ public final class SwiftTargetBuildDescription {
     /// a target is built.
     public let toolsVersion: ToolsVersion
 
-    /// The build parameters.
+    /// The build parameters for this target.
     let buildParameters: BuildParameters
 
     /// Path to the temporary directory for this target.
@@ -63,9 +63,10 @@ public final class SwiftTargetBuildDescription {
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
         if let bundleName = target.underlying.potentialBundleName, needsResourceBundle {
-            return self.buildParameters.bundlePath(named: bundleName)
+            let suffix = self.buildParameters.suffix
+            return self.buildParameters.bundlePath(named: bundleName + suffix)
         } else {
-            return .none
+            return nil
         }
     }
 
@@ -112,7 +113,8 @@ public final class SwiftTargetBuildDescription {
     }
 
     var modulesPath: AbsolutePath {
-        return self.buildParameters.buildPath.appending(component: "Modules")
+        let suffix = self.buildParameters.suffix
+        return self.buildParameters.buildPath.appending(component: "Modules\(suffix)")
     }
 
     /// The path to the swiftmodule file after compilation.
@@ -121,7 +123,7 @@ public final class SwiftTargetBuildDescription {
         let triple = buildParameters.triple
         let allowLinkingAgainstExecutables = (triple.isDarwin() || triple.isLinux() || triple.isWindows()) && self.toolsVersion >= .v5_5
         let dirPath = (target.type == .executable && !allowLinkingAgainstExecutables) ? self.tempsPath : self.modulesPath
-        return dirPath.appending(component: self.target.c99name + ".swiftmodule")
+        return dirPath.appending(component: "\(self.target.c99name).swiftmodule")
     }
 
     /// The path to the wrapped swift module which is created using the modulewrap tool. This is required
@@ -131,7 +133,7 @@ public final class SwiftTargetBuildDescription {
         self.tempsPath.appending(component: self.target.c99name + ".swiftmodule.o")
     }
 
-    /// The path to the swifinterface file after compilation.
+    /// The path to the swiftinterface file after compilation.
     var parseableModuleInterfaceOutputPath: AbsolutePath {
         self.modulesPath.appending(component: self.target.c99name + ".swiftinterface")
     }
@@ -233,7 +235,7 @@ public final class SwiftTargetBuildDescription {
     public let prebuildCommandResults: [PrebuildCommandResult]
 
     /// Any macro products that this target requires to build.
-    public let requiredMacroProducts: [ResolvedProduct]
+    public let requiredMacroProducts: [ProductBuildDescription]
 
     /// ObservabilityScope with which to emit diagnostics
     private let observabilityScope: ObservabilityScope
@@ -242,7 +244,7 @@ public final class SwiftTargetBuildDescription {
     private let shouldGenerateTestObservation: Bool
 
     /// Whether to disable sandboxing (e.g. for macros).
-    private let disableSandbox: Bool
+    private let shouldDisableSandbox: Bool
 
     /// Create a new target description with target and build parameters.
     init(
@@ -253,10 +255,10 @@ public final class SwiftTargetBuildDescription {
         buildParameters: BuildParameters,
         buildToolPluginInvocationResults: [BuildToolPluginInvocationResult] = [],
         prebuildCommandResults: [PrebuildCommandResult] = [],
-        requiredMacroProducts: [ResolvedProduct] = [],
+        requiredMacroProducts: [ProductBuildDescription] = [],
         testTargetRole: TestTargetRole? = nil,
         shouldGenerateTestObservation: Bool = false,
-        disableSandbox: Bool,
+        shouldDisableSandbox: Bool,
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope
     ) throws {
@@ -269,6 +271,7 @@ public final class SwiftTargetBuildDescription {
         self.target = target
         self.toolsVersion = toolsVersion
         self.buildParameters = buildParameters
+
         // Unless mentioned explicitly, use the target type to determine if this is a test target.
         if let testTargetRole {
             self.testTargetRole = testTargetRole
@@ -278,21 +281,21 @@ public final class SwiftTargetBuildDescription {
             self.testTargetRole = nil
         }
 
-        self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
+        self.tempsPath = target.tempsPath(self.buildParameters)
         self.derivedSources = Sources(paths: [], root: self.tempsPath.appending("DerivedSources"))
         self.buildToolPluginInvocationResults = buildToolPluginInvocationResults
         self.prebuildCommandResults = prebuildCommandResults
         self.requiredMacroProducts = requiredMacroProducts
         self.shouldGenerateTestObservation = shouldGenerateTestObservation
-        self.disableSandbox = disableSandbox
+        self.shouldDisableSandbox = shouldDisableSandbox
         self.fileSystem = fileSystem
         self.observabilityScope = observabilityScope
 
         (self.pluginDerivedSources, self.pluginDerivedResources) = ModulesGraph.computePluginGeneratedFiles(
             target: target,
             toolsVersion: toolsVersion,
             additionalFileRules: additionalFileRules,
-            buildParameters: buildParameters,
+            buildParameters: self.buildParameters,
             buildToolPluginInvocationResults: buildToolPluginInvocationResults,
             prebuildCommandResults: prebuildCommandResults,
             observabilityScope: observabilityScope
@@ -332,7 +335,10 @@ public final class SwiftTargetBuildDescription {
             return
         }
 
-        guard self.buildParameters.triple.isDarwin(), self.buildParameters.testingParameters.experimentalTestOutput else {
+        guard 
+            self.buildParameters.triple.isDarwin() &&
+            self.buildParameters.testingParameters.experimentalTestOutput
+        else {
             return
         }
 
@@ -412,21 +418,25 @@ public final class SwiftTargetBuildDescription {
 
         #if BUILD_MACROS_AS_DYLIBS
         self.requiredMacroProducts.forEach { macro in
-            args += ["-Xfrontend", "-load-plugin-library", "-Xfrontend", self.buildParameters.binaryPath(for: macro).pathString]
+            args += ["-Xfrontend", "-load-plugin-library", "-Xfrontend", macro.binaryPath.pathString]
         }
         #else
         try self.requiredMacroProducts.forEach { macro in
-            if let macroTarget = macro.targets.first {
-                let executablePath = try self.buildParameters.binaryPath(for: macro).pathString
+            if let macroTarget = macro.product.targets.first {
+                let executablePath = try macro.binaryPath.pathString
                 args += ["-Xfrontend", "-load-plugin-executable", "-Xfrontend", "\(executablePath)#\(macroTarget.c99name)"]
             } else {
-                throw InternalError("macro product \(macro.name) has no targets") // earlier validation should normally catch this
+                throw InternalError("macro product \(macro.product.name) has no targets") // earlier validation should normally catch this
             }
         }
         #endif
 
-        if self.disableSandbox {
-            let toolchainSupportsDisablingSandbox = DriverSupport.checkSupportedFrontendFlags(flags: ["-disable-sandbox"], toolchain: self.buildParameters.toolchain, fileSystem: fileSystem)
+        if self.shouldDisableSandbox {
+            let toolchainSupportsDisablingSandbox = DriverSupport.checkSupportedFrontendFlags(
+                flags: ["-disable-sandbox"],
+                toolchain: self.buildParameters.toolchain,
+                fileSystem: fileSystem
+            )
             if toolchainSupportsDisablingSandbox {
                 args += ["-disable-sandbox"]
             } else {
@@ -443,7 +453,7 @@ public final class SwiftTargetBuildDescription {
     /// The arguments needed to compile this target.
     public func compileArguments() throws -> [String] {
         var args = [String]()
-        args += try self.buildParameters.targetTripleArgs(for: self.target)
+        args += try self.buildParameters.tripleArgs(for: self.target)
 
         // pass `-v` during verbose builds.
         if self.buildParameters.outputParameters.isVerbose {
@@ -818,7 +828,7 @@ public final class SwiftTargetBuildDescription {
         // Include path for the toolchain's copy of SwiftSyntax.
         #if BUILD_MACROS_AS_DYLIBS
         if target.type == .macro {
-            flags += try ["-I", self.buildParameters.toolchain.hostLibDir.pathString]
+            flags += try ["-I", self.defaultBuildParameters.toolchain.hostLibDir.pathString]
         }
         #endif
 

Sources/Build/BuildManifest/LLBuildManifestBuilder+Clang.swift

@@ -93,7 +93,7 @@ extension LLBuildManifestBuilder {
         let additionalInputs = try addBuildToolPlugins(.clang(target))
 
         // Create a phony node to represent the entire target.
-        let targetName = target.target.getLLBuildTargetName(config: target.buildParameters.buildConfig)
+        let targetName = target.llbuildTargetName
         let output: Node = .virtual(targetName)
 
         self.manifest.addNode(output, toTarget: targetName)