Implement A3 High network blocking script as startup-script feature

modules/scripts/startup-script/README.md

@@ -320,6 +320,7 @@ No modules.
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes |
 | <a name="input_docker"></a> [docker](#input\_docker) | Install and configure Docker | <pre>object({<br/>    enabled        = optional(bool, false)<br/>    world_writable = optional(bool, false)<br/>    daemon_config  = optional(string, "")<br/>  })</pre> | <pre>{<br/>  "enabled": false<br/>}</pre> | no |
 | <a name="input_enable_docker_world_writable"></a> [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no |
+| <a name="input_enable_gpu_network_wait_online"></a> [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no |
 | <a name="input_gcs_bucket_path"></a> [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no |
 | <a name="input_http_no_proxy"></a> [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no |
 | <a name="input_http_proxy"></a> [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no |

modules/scripts/startup-script/files/install_gpu_network_wait_online.yml

@@ -0,0 +1,54 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Wait until all network interfaces are online
+  hosts: all
+  become: true
+  tasks:
+  - name: Create SystemD service for A3 High networking
+    when: ansible_os_family == "Debian"
+    ansible.builtin.copy:
+      dest: /etc/systemd/system/delay-a3-high.service
+      owner: root
+      group: root
+      mode: "0644"
+      content: |
+        [Unit]
+        Description=Delay A3 High boot until all network interfaces are routable
+        After=network-online.target
+        Wants=network-online.target
+        Before=google-startup-scripts.service
+
+        [Service]
+        ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetata/v1/instance/machine-type | grep -q "/a3-highgpu-8g$"'
+        ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -i enp0s12 -i enp6s0 -i enp12s0 -i enp134s0 -i enp140s0 -o routable --timeout=180
+        ExecStartPost=/bin/sleep 30
+
+        [Install]
+        WantedBy=multi-user.target
+    notify:
+    - Reload SystemD
+    - Enable A3 High delay
+  handlers:
+  - name: Reload SystemD
+    ansible.builtin.systemd:
+      daemon_reload: true
+  post_tasks:
+  - name: Enable A3 High delay
+    # by the time this startup-script executes, we don't care if it's started
+    # just enabled
+    ansible.builtin.systemd_service:
+      name: delay-a3-high
+      enabled: true

modules/scripts/startup-script/main.tf

@@ -143,6 +143,15 @@ locals {
     },
   ]
 
+  gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [
+    {
+      type        = "ansible-local"
+      destination = "install_gpu_network_wait_online.yml"
+      content     = file("${path.module}/files/install_gpu_network_wait_online.yml")
+      args        = ""
+    },
+  ]
+
   local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint))
   raid_setup = !local.local_ssd_filesystem_enabled ? [] : [
     {
@@ -163,6 +172,7 @@ locals {
     local.configure_ssh,
     var.docker.enabled,
     var.managed_lustre.enabled,
+    var.enable_gpu_network_wait_online,
     local.local_ssd_filesystem_enabled
   ])
 
@@ -192,6 +202,7 @@ locals {
     local.managed_lustre_runner,
     local.configure_ssh_runners,
     local.docker_runner,
+    local.gpu_network_wait_online_runner,
     var.runners
   )
 

modules/scripts/startup-script/variables.tf

@@ -290,3 +290,9 @@ variable "set_ofi_cloud_rdma_tunables" {
   type        = bool
   default     = false
 }
+
+variable "enable_gpu_network_wait_online" {
+  description = "Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution)"
+  type        = bool
+  default     = false
+}