[8.4.0] Introduce shorten_virtual_includes cc toolchain feature (#26532)

Fixes #18683

Related: protocolbuffers/protobuf#20085

This change is a rework of
#26005 to enable short virtual
includes based on a cc feature, therefore we can limit the change to
only MSVC compiler on Windows.

RELNOTES: If a cc toolchain feature named `shorten_virtual_includes` is
enabled, virtual include header files are linked under
`bin/_virtual_includes/<hash of target path>` instead of `bin/<target
package path>/_virtual_includes/<target name>`. This shortens the
virtual include paths which is critical for mitigating long path issue
with MSVC on Windows.

Closes #26528.

PiperOrigin-RevId: 781975309
Change-Id: Ia573a5f25707ad2462aa3a4459fc66db1779df36

---------

Co-authored-by: Googler <[email protected]>

Author: meteorcloudy

site/en/docs/cc-toolchain-config-reference.md

@@ -1093,6 +1093,13 @@ conditions.
        features below.
      </td>
     </tr>
+  <tr>
+     <td><strong><code>shorten_virtual_includes</code></strong>
+     </td>
+     <td>
+       If enabled, virtual include header files are linked under <code>bin/_virtual_includes/&lt;hash of target path&gt;</code> instead of <code>bin/&lt;target package path&gt;/_virtual_includes/&lt;target name&gt;</code>. Useful on Windows to avoid long path issue with MSVC.
+     </td>
+    </tr>
 </table>
 
 #### Legacy features patching logic {:#legacy-features-patching-logic}

src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java

@@ -387,6 +387,12 @@ public static ToolchainTypeRequirement ccToolchainTypeRequirement(RuleDefinition
   /** A string constant for a feature that, if enabled, disables .d file handling. */
   public static final String NO_DOTD_FILE = "no_dotd_file";
 
+  /**
+   * A string constant for a feature that, if enabled, shortens the virtual include paths via
+   * hashing.
+   */
+  public static final String SHORTEN_VIRTUAL_INCLUDES = "shorten_virtual_includes";
+
   /*
    * A string constant for the fdo_instrument feature.
    */

src/main/starlark/builtins_bzl/common/cc/cc_compilation_helper.bzl

@@ -15,7 +15,11 @@
 """Compilation helper for C++ rules."""
 
 load(":common/cc/cc_common.bzl", "cc_common")
-load(":common/cc/cc_helper.bzl", "cc_helper")
+load(
+    ":common/cc/cc_helper_internal.bzl",
+    "package_source_root",
+    "repository_exec_path",
+)
 load(":common/cc/semantics.bzl", "USE_EXEC_ROOT_FOR_VIRTUAL_INCLUDES_SYMLINKS")
 load(":common/paths.bzl", "paths")
 
@@ -55,7 +59,8 @@ def _compute_public_headers(
         label,
         binfiles_dir,
         non_module_map_headers,
-        is_sibling_repository_layout):
+        is_sibling_repository_layout,
+        shorten_virtual_includes):
     if include_prefix:
         if not paths.is_normalized(include_prefix, False):
             fail("include prefix should not contain uplevel references: " + include_prefix)
@@ -107,7 +112,11 @@ def _compute_public_headers(
 
     module_map_headers = []
     virtual_to_original_headers_list = []
-    virtual_include_dir = paths.join(paths.join(cc_helper.package_source_root(label.workspace_name, label.package, is_sibling_repository_layout), _VIRTUAL_INCLUDES_DIR), label.name)
+    source_package_path = package_source_root(label.workspace_name, label.package, is_sibling_repository_layout)
+    if shorten_virtual_includes:
+        virtual_include_dir = paths.join(_VIRTUAL_INCLUDES_DIR, "%x" % hash(paths.join(source_package_path, label.name)))
+    else:
+        virtual_include_dir = paths.join(source_package_path, _VIRTUAL_INCLUDES_DIR, label.name)
     for original_header in public_headers_artifacts:
         repo_relative_path = _repo_relative_path(original_header)
         if not repo_relative_path.startswith(strip_prefix):
@@ -144,7 +153,7 @@ def _generates_header_module(feature_configuration, public_headers, private_head
            generate_action
 
 def _header_module_artifact(actions, label, is_sibling_repository_layout, suffix, extension):
-    object_dir = paths.join(paths.join(cc_helper.package_source_root(label.workspace_name, label.package, is_sibling_repository_layout), "_objs"), label.name)
+    object_dir = paths.join(paths.join(package_source_root(label.workspace_name, label.package, is_sibling_repository_layout), "_objs"), label.name)
     base_name = label.name.split("/")[-1]
     output_path = paths.join(object_dir, base_name + suffix + extension)
     return actions.declare_shareable_artifact(output_path)
@@ -219,11 +228,12 @@ def _init_cc_compilation_context(
     # we might pick up stale generated files.
     sibling_repo_layout = config.is_sibling_repository_layout()
     repo_name = label.workspace_name
-    repo_path = cc_helper.repository_exec_path(repo_name, sibling_repo_layout)
+    repo_path = repository_exec_path(repo_name, sibling_repo_layout)
     gen_include_dir = _include_dir(genfiles_dir, repo_path, sibling_repo_layout)
     bin_include_dir = _include_dir(binfiles_dir, repo_path, sibling_repo_layout)
     quote_include_dirs_for_context = [repo_path, gen_include_dir, bin_include_dir] + quote_include_dirs
     external = repo_name != "" and _enabled(feature_configuration, "external_include_paths")
+    shorten_virtual_includes = _enabled(feature_configuration, "shorten_virtual_includes")
     external_include_dirs = []
     declared_include_srcs = []
 
@@ -252,6 +262,7 @@ def _init_cc_compilation_context(
         binfiles_dir,
         non_module_map_headers,
         sibling_repo_layout,
+        shorten_virtual_includes,
     )
     if public_headers.virtual_include_path:
         if external:
@@ -289,6 +300,7 @@ def _init_cc_compilation_context(
         binfiles_dir,
         non_module_map_headers,
         sibling_repo_layout,
+        shorten_virtual_includes,
     )
 
     separate_module = None

src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl

@@ -20,6 +20,8 @@ load(
     "is_versioned_shared_library_extension_valid",
     "should_create_per_object_debug_info",
     _artifact_category = "artifact_category",
+    _package_source_root = "package_source_root",
+    _repository_exec_path = "repository_exec_path",
 )
 load(":common/cc/cc_info.bzl", "CcInfo")
 load(":common/objc/objc_common.bzl", "objc_common")
@@ -1039,29 +1041,9 @@ def _report_invalid_options(cc_toolchain, cpp_config):
     if cpp_config.grte_top() != None and cc_toolchain.sysroot == None:
         fail("The selected toolchain does not support setting --grte_top (it doesn't specify builtin_sysroot).")
 
-def _is_repository_main(repository):
-    return repository == ""
-
-def _repository_exec_path(repository, sibling_repository_layout):
-    if _is_repository_main(repository):
-        return ""
-    prefix = "external"
-    if sibling_repository_layout:
-        prefix = ".."
-    if repository.startswith("@"):
-        repository = repository[1:]
-    return paths.get_relative(prefix, repository)
-
 def _package_exec_path(ctx, package, sibling_repository_layout):
     return paths.get_relative(_repository_exec_path(ctx.label.workspace_name, sibling_repository_layout), package)
 
-def _package_source_root(repository, package, sibling_repository_layout):
-    if _is_repository_main(repository) or sibling_repository_layout:
-        return package
-    if repository.startswith("@"):
-        repository = repository[1:]
-    return paths.get_relative(paths.get_relative("external", repository), package)
-
 def _system_include_dirs(ctx, additional_make_variable_substitutions):
     result = []
     sibling_repository_layout = ctx.configuration.is_sibling_repository_layout()

src/main/starlark/builtins_bzl/common/cc/cc_helper_internal.bzl

@@ -18,6 +18,8 @@ Utility functions for C++ rules that don't depend on cc_common.
 Only use those within C++ implementation. The others need to go through cc_common.
 """
 
+load(":common/paths.bzl", "paths")
+
 cc_common_internal = _builtins.internal.cc_common
 
 CREATE_COMPILE_ACTION_API_ALLOWLISTED_PACKAGES = [("", "devtools/rust/cc_interop"), ("", "third_party/crubit")]
@@ -126,6 +128,47 @@ def is_versioned_shared_library(file):
         return False
     return is_versioned_shared_library_extension_valid(file.basename)
 
+def _is_repository_main(repository):
+    return repository == ""
+
+def package_source_root(repository, package, sibling_repository_layout):
+    """
+    Determines the source root for a given repository and package.
+
+    Args:
+      repository: The repository to get the source root for.
+      package: The package to get the source root for.
+      sibling_repository_layout: Whether the repository layout is a sibling repository layout.
+
+    Returns:
+      The source root for the given repository and package.
+    """
+    if _is_repository_main(repository) or sibling_repository_layout:
+        return package
+    if repository.startswith("@"):
+        repository = repository[1:]
+    return paths.get_relative(paths.get_relative("external", repository), package)
+
+def repository_exec_path(repository, sibling_repository_layout):
+    """
+    Determines the exec path for a given repository.
+
+    Args:
+      repository: The repository to get the exec path for.
+      sibling_repository_layout: Whether the repository layout is a sibling repository layout.
+
+    Returns:
+      The exec path for the given repository.
+    """
+    if _is_repository_main(repository):
+        return ""
+    prefix = "external"
+    if sibling_repository_layout:
+        prefix = ".."
+    if repository.startswith("@"):
+        repository = repository[1:]
+    return paths.get_relative(prefix, repository)
+
 def use_pic_for_binaries(cpp_config, feature_configuration):
     """
     Returns whether binaries must be compiled with position independent code.

src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl

@@ -122,6 +122,7 @@ _FEATURE_NAMES = struct(
     cpp_compile_with_requirements = "cpp_compile_with_requirements",
     no_copts_tokenization = "no_copts_tokenization",
     generate_linkmap = "generate_linkmap",
+    shorten_virtual_includes = "shorten_virtual_includes",
 )
 
 _cpp_modules_feature = feature(
@@ -142,6 +143,11 @@ _supports_dynamic_linker_feature = feature(
     enabled = True,
 )
 
+_shorten_virtual_includes_feature = feature(
+    name = _FEATURE_NAMES.shorten_virtual_includes,
+    enabled = True,
+)
+
 _supports_interface_shared_libraries_feature = feature(
     name = _FEATURE_NAMES.supports_interface_shared_libraries,
     enabled = True,
@@ -1451,6 +1457,7 @@ _feature_name_to_feature = {
     _FEATURE_NAMES.cpp_compile_with_requirements: _cpp_compile_with_requirements,
     _FEATURE_NAMES.generate_pdb_file: _generate_pdb_file_feature,
     _FEATURE_NAMES.generate_linkmap: _generate_linkmap_feature,
+    _FEATURE_NAMES.shorten_virtual_includes: _shorten_virtual_includes_feature,
     "header_modules_feature_configuration": _header_modules_feature_configuration,
     "env_var_feature_configuration": _env_var_feature_configuration,
     "host_and_nonhost_configuration": _host_and_nonhost_configuration,

src/test/java/com/google/devtools/build/lib/rules/cpp/CcCommonTest.java

@@ -1079,6 +1079,37 @@ public void testIncludeManglingSmoke() throws Exception {
                 .getRelative("third_party/a/_virtual_includes/a"));
   }
 
+  @Test
+  public void testIncludeManglingSmokeWithShortendVirtualIncludes() throws Exception {
+    getAnalysisMock()
+        .ccSupport()
+        .setupCcToolchainConfig(
+            mockToolsConfig,
+            CcToolchainConfig.builder().withFeatures(CppRuleClasses.SHORTEN_VIRTUAL_INCLUDES));
+    scratch.file(
+        "third_party/a/BUILD",
+        """
+        licenses(["notice"])
+
+        cc_library(
+            name = "a",
+            hdrs = ["v1/b/c.h"],
+            include_prefix = "lib",
+            strip_include_prefix = "v1",
+        )
+        """);
+
+    ConfiguredTarget lib = getConfiguredTarget("//third_party/a");
+    CcCompilationContext ccCompilationContext = lib.get(CcInfo.PROVIDER).getCcCompilationContext();
+    assertThat(ActionsTestUtil.prettyArtifactNames(ccCompilationContext.getDeclaredIncludeSrcs()))
+        .containsExactly("_virtual_includes/207132b2/lib/b/c.h", "third_party/a/v1/b/c.h");
+    assertThat(ccCompilationContext.getIncludeDirs())
+        .containsExactly(
+            getTargetConfiguration()
+                .getBinFragment(RepositoryName.MAIN)
+                .getRelative("_virtual_includes/207132b2"));
+  }
+
   @Test
   public void testUpLevelReferencesInIncludeMangling() throws Exception {
     scratch.file(