chore: Support for Azure Container Registry (ACR) Credential Provider

docs/topics/azure-acr-credential-provider.md

@@ -0,0 +1,48 @@
+# Azure Container Registry (ACR) Credential Provider
+
+## Overview
+
+Starting with Kubernetes v1.29, you **must** use [Out-of-Tree Credential Providers][KEP] to pull container images from private Azure Container Registry (ACR) instances. The legacy `--azure-container-registry-config` kubelet flag has been deprecated and removed. See [Cloud Provider Azure][CPA] documentation for more details.
+
+## What This Means for Your Cluster
+
+When you deploy an AKS Engine cluster, the migration to ACR credential providers happens automatically:
+
+ **Automatic Migration**: AKS Engine removes the deprecated `--azure-container-registry-config` flag  
+ **New Configuration**: Replaces it with modern credential provider flags:
+- `--image-credential-provider-bin-dir`
+- `--image-credential-provider-config`
+
+> **No Action Required**: This transition is transparent - your cluster will continue to authenticate with ACR without any manual intervention.
+
+## How It Works
+
+### Configuration File Location
+
+The credential provider uses a configuration file automatically deployed to:
+```
+/var/lib/kubelet/credential-provider-config.yaml
+```
+
+This file is **automatically provisioned** to all nodes (both master and worker) in your cluster.
+
+### Default Configuration
+
+The default configuration handles authentication for all Azure Container Registry endpoints:
+
+```yaml
+kind: CredentialProviderConfig
+apiVersion: kubelet.config.k8s.io/v1
+providers:
+  - name: azure-acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"    # Azure Public Cloud
+      - "*.azurecr.cn"    # Azure China Cloud  
+      - "*.azurecr.de"    # Azure Germany Cloud
+      - "*.azurecr.us"    # Azure US Government Cloud
+```
+
+> **Reference**: View the complete default configuration file at [`credential-provider-config.yaml`](../../parts/k8s/cloud-init/artifacts/credential-provider-config.yaml)
+
+[KEP]: https://github.com/kubernetes/enhancements/tree/master/keps/sig-cloud-provider/2133-out-of-tree-credential-provider
+[CPA]: https://cloud-provider-azure.sigs.k8s.io/topics/credential-provider/

docs/topics/clusterdefinitions.md

@@ -461,6 +461,8 @@ Below is a list of kubelet options that AKS Engine will configure by default:
 | "--register-with-taints"              | "node-role.kubernetes.io/control-plane=true:NoSchedule" (`masterProfile` only; Note: you may add your own master-specific taints in the `kubeletConfig` under `masterProfile`, which will augment the built-in "node-role.kubernetes.io/control-plane=true:NoSchedule" taint, which will always be present.)            |
 | "--runtime-request-timeout"           | "15m" if in a containerd configuration, otherwise this configuration is not passed to kubelet runtime                                                                                                                                                                                                     |
 | "--container-runtime-endpoint"        | "unix:///run/containerd/containerd.sock" if in a containerd configuration, otherwise this configuration is not passed to kubelet runtime                                                                                                                                                                  |
+| "--image-credential-provider-config"   | "/var/lib/kubelet/credential-provider-config.yaml"  (default content overridable through _CustomFiles_) |
+
 
 Below is a list of kubelet options that are _not_ currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster:
 
@@ -481,6 +483,7 @@ Below is a list of kubelet options that are _not_ currently user-configurable, e
 | "--tls-private-key-file"        | "/etc/kubernetes/certs/kubeletserver.key"          |
 | "--v"                           | "2"                                                |
 | "--volume-plugin-dir"           | "/etc/kubernetes/volumeplugins"                    |
+| "--image-credential-provider-bin-dir"  | "/var/lib/kubelet/credential-provider"                                                  |
 
 <a name="feat-controller-manager-config"></a>
 

examples/e2e-tests/kubernetes/release/default/definition-no-vnet.json

@@ -11,7 +11,10 @@
                         "name": "azuredisk-csi-driver",
                         "enabled": true
                     }
-                ]
+                ],
+                "kubeletConfig": {
+                    "--image-credential-provider-config": "/var/lib/kubelet/credential-provider-config.yaml"
+                }
             }
         },
         "masterProfile": {

parts/k8s/cloud-init/artifacts/credential-provider-config.yaml

@@ -0,0 +1,13 @@
+kind: CredentialProviderConfig
+apiVersion: kubelet.config.k8s.io/v1
+providers:
+  - name: azure-acr-credential-provider
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    defaultCacheDuration: 10m
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+    args:
+      - /etc/kubernetes/azure.json

parts/k8s/cloud-init/artifacts/cse_install.sh

@@ -3,6 +3,7 @@
 CNI_CFG_DIR="/etc/cni/net.d"
 CNI_BIN_DIR="/opt/cni/bin"
 CNI_DL_DIR="/opt/cni/downloads"
+ACR_DL_DIR="/var/lib/kubelet/credential-provider"
 CTRD_DL_DIR="/opt/containerd/downloads"
 APMZ_DL_DIR="/opt/apmz/downloads"
 UBUNTU_RELEASE=$(lsb_release -r -s)
@@ -143,6 +144,12 @@ downloadAzureCNI() {
   CNI_TGZ_TMP=${VNET_CNI_PLUGINS_URL##*/}
   retrycmd_get_tarball 120 5 "$CNI_DL_DIR/${CNI_TGZ_TMP}" ${VNET_CNI_PLUGINS_URL} || exit 41
 }
+downloadACR() {
+  mkdir -p $ACR_DL_DIR
+  retrycmd 30 5 60 curl ${PROVIDER_ARTIFACT} -fSL -o "$ACR_DL_DIR/azure-acr-credential-provider" || exit 41
+  chown -R root:root $ACR_DL_DIR
+  chmod 755 $ACR_DL_DIR/azure-acr-credential-provider
+}
 ensureAPMZ() {
   local ver=$1 v
   local d="$APMZ_DL_DIR/$ver"

parts/k8s/cloud-init/nodecustomdata.yml

@@ -367,7 +367,6 @@ write_files:
     {{- /* Ensure that container traffic can't connect to internal Azure IP endpoint */}}
     iptables -I FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP
     #EOF
-
 {{- if IsCustomCloudProfile}}
 - path: "/etc/kubernetes/azurestackcloud.json"
   permissions: "0600"

pkg/api/defaults-kubelet.go

@@ -15,20 +15,21 @@ import (
 func (cs *ContainerService) setKubeletConfig(isUpgrade bool) {
 	o := cs.Properties.OrchestratorProfile
 	staticLinuxKubeletConfig := map[string]string{
-		"--address":                     "0.0.0.0",
-		"--allow-privileged":            "true",
-		"--anonymous-auth":              "false",
-		"--authorization-mode":          "Webhook",
-		"--client-ca-file":              "/etc/kubernetes/certs/ca.crt",
-		"--pod-manifest-path":           "/etc/kubernetes/manifests",
-		"--cluster-dns":                 o.KubernetesConfig.DNSServiceIP,
-		"--cgroups-per-qos":             "true",
-		"--kubeconfig":                  "/var/lib/kubelet/kubeconfig",
-		"--keep-terminated-pod-volumes": "false",
-		"--tls-cert-file":               "/etc/kubernetes/certs/kubeletserver.crt",
-		"--tls-private-key-file":        "/etc/kubernetes/certs/kubeletserver.key",
-		"--v":                           "2",
-		"--volume-plugin-dir":           "/etc/kubernetes/volumeplugins",
+		"--address":                           "0.0.0.0",
+		"--allow-privileged":                  "true",
+		"--anonymous-auth":                    "false",
+		"--authorization-mode":                "Webhook",
+		"--client-ca-file":                    "/etc/kubernetes/certs/ca.crt",
+		"--pod-manifest-path":                 "/etc/kubernetes/manifests",
+		"--cluster-dns":                       o.KubernetesConfig.DNSServiceIP,
+		"--cgroups-per-qos":                   "true",
+		"--kubeconfig":                        "/var/lib/kubelet/kubeconfig",
+		"--keep-terminated-pod-volumes":       "false",
+		"--tls-cert-file":                     "/etc/kubernetes/certs/kubeletserver.crt",
+		"--tls-private-key-file":              "/etc/kubernetes/certs/kubeletserver.key",
+		"--v":                                 "2",
+		"--volume-plugin-dir":                 "/etc/kubernetes/volumeplugins",
+		"--image-credential-provider-bin-dir": "/var/lib/kubelet/credential-provider",
 	}
 
 	for key := range staticLinuxKubeletConfig {
@@ -65,6 +66,8 @@ func (cs *ContainerService) setKubeletConfig(isUpgrade bool) {
 
 	// Add Windows-specific overrides
 	// Eventually paths should not be hardcoded here. They should be relative to $global:KubeDir in the PowerShell script
+	staticWindowsKubeletConfig["--image-credential-provider-config"] = "c:\\k\\credential-provider\\credential-provider-config.yaml"
+	staticWindowsKubeletConfig["--image-credential-provider-bin-dir"] = "c:\\k\\credential-provider"
 	staticWindowsKubeletConfig["--pod-infra-container-image"] = "kubletwin/pause"
 	staticWindowsKubeletConfig["--kubeconfig"] = "c:\\k\\config"
 	staticWindowsKubeletConfig["--cloud-config"] = "c:\\k\\azure.json"
@@ -103,6 +106,7 @@ func (cs *ContainerService) setKubeletConfig(isUpgrade bool) {
 		"--tls-cipher-suites":                 TLSStrongCipherSuitesKubelet,
 		"--healthz-port":                      DefaultKubeletHealthzPort,
 		"--seccomp-default":                   "true",
+		"--image-credential-provider-config":  "/var/lib/kubelet/credential-provider-config.yaml",
 	}
 
 	// Set --non-masquerade-cidr if ip-masq-agent is disabled on AKS or

pkg/api/defaults-kubelet_test.go

@@ -107,6 +107,8 @@ func TestKubeletConfigDefaults(t *testing.T) {
 	delete(expected, "--register-with-taints")
 
 	windowsProfileKubeletConfig := cs.Properties.AgentPoolProfiles[1].KubernetesConfig.KubeletConfig
+	expected["--image-credential-provider-config"] = "c:\\k\\credential-provider\\credential-provider-config.yaml"
+	expected["--image-credential-provider-bin-dir"] = "c:\\k\\credential-provider"
 	expected["--pod-infra-container-image"] = "kubletwin/pause"
 	expected["--kubeconfig"] = "c:\\k\\config"
 	expected["--cloud-config"] = "c:\\k\\azure.json"
@@ -171,13 +173,16 @@ func TestKubeletConfigDefaults(t *testing.T) {
 
 	cs = CreateMockContainerService("testcluster", "", 3, 2, false)
 	// TODO test all default overrides
-	// Removed kubelet --azure-container-registry-config deprecated CLI flag
 	overrideVal := "/etc/override"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = map[string]string{
+		"--image-credential-provider-config": overrideVal,
+	}
 	cs.setKubeletConfig(false)
 	k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
-	for key := range map[string]string{"--azure-container-registry-config": overrideVal} {
-		if _, ok := k[key]; ok {
-			t.Fatal("got unexpected (removed) '--azure-container-registry-config' kubelet config value")
+	for key, val := range map[string]string{"--image-credential-provider-config": overrideVal} {
+		if k[key] != val {
+			t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
+				key, k[key], val)
 		}
 	}
 
@@ -202,6 +207,8 @@ func getDefaultLinuxKubeletConfig(cs *ContainerService) map[string]string {
 		"--anonymous-auth":                    "false",
 		"--authorization-mode":                "Webhook",
 		"--authentication-token-webhook":      "true",
+		"--image-credential-provider-config":  "/var/lib/kubelet/credential-provider-config.yaml",
+		"--image-credential-provider-bin-dir": "/var/lib/kubelet/credential-provider",
 		"--cadvisor-port":                     "", // Validate that we delete this key for >= 1.12 clusters
 		"--cgroups-per-qos":                   "true",
 		"--client-ca-file":                    "/etc/kubernetes/certs/ca.crt",
@@ -257,6 +264,8 @@ func TestKubeletConfigAzureStackDefaults(t *testing.T) {
 		"--anonymous-auth":                    "false",
 		"--authentication-token-webhook":      "true",
 		"--authorization-mode":                "Webhook",
+		"--image-credential-provider-config":  "/var/lib/kubelet/credential-provider-config.yaml",
+		"--image-credential-provider-bin-dir": "/var/lib/kubelet/credential-provider",
 		"--cadvisor-port":                     "", // Validate that we delete this key for >= 1.12 clusters
 		"--cgroups-per-qos":                   "true",
 		"--client-ca-file":                    "/etc/kubernetes/certs/ca.crt",
@@ -315,6 +324,8 @@ func TestKubeletConfigAzureStackDefaults(t *testing.T) {
 	}
 
 	windowsProfileKubeletConfig := cs.Properties.AgentPoolProfiles[1].KubernetesConfig.KubeletConfig
+	expected["--image-credential-provider-config"] = "c:\\k\\credential-provider\\credential-provider-config.yaml"
+	expected["--image-credential-provider-bin-dir"] = "c:\\k\\credential-provider"
 	expected["--pod-infra-container-image"] = "kubletwin/pause"
 	expected["--kubeconfig"] = "c:\\k\\config"
 	expected["--cloud-config"] = "c:\\k\\azure.json"
@@ -358,13 +369,16 @@ func TestKubeletConfigAzureStackDefaults(t *testing.T) {
 
 	cs = CreateMockContainerService("testcluster", "", 3, 2, false)
 	// TODO test all default overrides
-	// Removed kubelet --azure-container-registry-config deprecated CLI flag
 	overrideVal := "/etc/override"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = map[string]string{
+		"--image-credential-provider-config": overrideVal,
+	}
 	cs.setKubeletConfig(false)
 	k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
-	for key := range map[string]string{"--azure-container-registry-config": overrideVal} {
-		if _, ok := k[key]; ok {
-			t.Fatal("got unexpected (removed) '--azure-container-registry-config' kubelet config value")
+	for key, val := range map[string]string{"--image-credential-provider-config": overrideVal} {
+		if k[key] != val {
+			t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
+				key, k[key], val)
 		}
 	}
 
@@ -459,6 +473,34 @@ func TestKubeletConfigCloudConfig(t *testing.T) {
 	}
 }
 
+func TestKubeletConfigAzureContainerRegistryConfig(t *testing.T) {
+	// Test default value and custom value for --image-credential-provider-config
+	cs := CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.setKubeletConfig(false)
+	k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	if k["--image-credential-provider-config"] != "/var/lib/kubelet/credential-provider-config.yaml" {
+		t.Fatalf("got unexpected '--image-credential-provider-config' kubelet config default value: %s",
+			k["--image-credential-provider-config"])
+	}
+	if k["--image-credential-provider-bin-dir"] != "/var/lib/kubelet/credential-provider" {
+		t.Fatalf("got unexpected '--image-credential-provider-bin-dir' kubelet config default value: %s",
+			k["--image-credential-provider-bin-dir"])
+	}
+
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--image-credential-provider-config"] = "custom.json"
+	cs.setKubeletConfig(false)
+	k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	if k["--image-credential-provider-config"] != "custom.json" {
+		t.Fatalf("got unexpected '--image-credential-provider-config' kubelet config default value: %s",
+			k["--image-credential-provider-config"])
+	}
+	if k["--image-credential-provider-bin-dir"] != "/var/lib/kubelet/credential-provider" {
+		t.Fatalf("got unexpected '--image-credential-provider-bin-dir' kubelet config default value: %s",
+			k["--image-credential-provider-bin-dir"])
+	}
+}
+
 func TestKubeletConfigNetworkPlugin(t *testing.T) {
 	// Test NetworkPlugin = "kubenet"
 	cs := CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
@@ -863,6 +905,8 @@ func TestStaticWindowsConfig(t *testing.T) {
 
 	// Add Windows-specific overrides
 	// Eventually paths should not be hardcoded here. They should be relative to $global:KubeDir in the PowerShell script
+	expected["--image-credential-provider-config"] = "c:\\k\\credential-provider\\credential-provider-config.yaml"
+	expected["--image-credential-provider-bin-dir"] = "c:\\k\\credential-provider"
 	expected["--pod-infra-container-image"] = "kubletwin/pause"
 	expected["--kubeconfig"] = "c:\\k\\config"
 	expected["--cloud-config"] = "c:\\k\\azure.json"

pkg/api/defaults_test.go

@@ -128,17 +128,18 @@ func TestCertsAlreadyPresent(t *testing.T) {
 func TestSetMissingKubeletValues(t *testing.T) {
 	config := &KubernetesConfig{}
 	defaultKubeletConfig := map[string]string{
-		"--network-plugin":               "1",
-		"--pod-infra-container-image":    "2",
-		"--max-pods":                     "3",
-		"--eviction-hard":                "4",
-		"--node-status-update-frequency": "5",
-		"--image-gc-high-threshold":      "6",
-		"--image-gc-low-threshold":       "7",
-		"--non-masquerade-cidr":          "8",
-		"--pod-max-pids":                 "10",
-		"--cloud-provider":               "azure",
-		"--cloud-config":                 "/etc/kubernetes/azure.json",
+		"--network-plugin":                   "1",
+		"--pod-infra-container-image":        "2",
+		"--max-pods":                         "3",
+		"--eviction-hard":                    "4",
+		"--node-status-update-frequency":     "5",
+		"--image-gc-high-threshold":          "6",
+		"--image-gc-low-threshold":           "7",
+		"--non-masquerade-cidr":              "8",
+		"--pod-max-pids":                     "10",
+		"--cloud-provider":                   "azure",
+		"--cloud-config":                     "/etc/kubernetes/azure.json",
+		"--image-credential-provider-config": "/var/lib/kubelet/credential-provider-config.yaml",
 	}
 	setMissingKubeletValues(config, defaultKubeletConfig)
 	for key, val := range defaultKubeletConfig {
@@ -155,17 +156,18 @@ func TestSetMissingKubeletValues(t *testing.T) {
 		},
 	}
 	expectedResult := map[string]string{
-		"--network-plugin":               "a",
-		"--pod-infra-container-image":    "b",
-		"--max-pods":                     "3",
-		"--eviction-hard":                "4",
-		"--node-status-update-frequency": "5",
-		"--image-gc-high-threshold":      "6",
-		"--image-gc-low-threshold":       "7",
-		"--non-masquerade-cidr":          "8",
-		"--pod-max-pids":                 "10",
-		"--cloud-provider":               "",
-		"--cloud-config":                 "/etc/kubernetes/azure.json",
+		"--network-plugin":                   "a",
+		"--pod-infra-container-image":        "b",
+		"--max-pods":                         "3",
+		"--eviction-hard":                    "4",
+		"--node-status-update-frequency":     "5",
+		"--image-gc-high-threshold":          "6",
+		"--image-gc-low-threshold":           "7",
+		"--non-masquerade-cidr":              "8",
+		"--pod-max-pids":                     "10",
+		"--cloud-provider":                   "",
+		"--cloud-config":                     "/etc/kubernetes/azure.json",
+		"--image-credential-provider-config": "/var/lib/kubelet/credential-provider-config.yaml",
 	}
 	setMissingKubeletValues(config, defaultKubeletConfig)
 	for key, val := range expectedResult {
@@ -176,22 +178,24 @@ func TestSetMissingKubeletValues(t *testing.T) {
 
 	config = &KubernetesConfig{
 		KubeletConfig: map[string]string{
-			"--cloud-provider": "",
-			"--cloud-config":   "",
+			"--cloud-provider":                   "",
+			"--cloud-config":                     "",
+			"--image-credential-provider-config": "",
 		},
 	}
 	expectedResult = map[string]string{
-		"--network-plugin":               "1",
-		"--pod-infra-container-image":    "2",
-		"--max-pods":                     "3",
-		"--eviction-hard":                "4",
-		"--node-status-update-frequency": "5",
-		"--image-gc-high-threshold":      "6",
-		"--image-gc-low-threshold":       "7",
-		"--non-masquerade-cidr":          "8",
-		"--pod-max-pids":                 "10",
-		"--cloud-provider":               "",
-		"--cloud-config":                 "",
+		"--network-plugin":                   "1",
+		"--pod-infra-container-image":        "2",
+		"--max-pods":                         "3",
+		"--eviction-hard":                    "4",
+		"--node-status-update-frequency":     "5",
+		"--image-gc-high-threshold":          "6",
+		"--image-gc-low-threshold":           "7",
+		"--non-masquerade-cidr":              "8",
+		"--pod-max-pids":                     "10",
+		"--cloud-provider":                   "",
+		"--cloud-config":                     "",
+		"--image-credential-provider-config": "",
 	}
 	setMissingKubeletValues(config, defaultKubeletConfig)
 	for key, val := range expectedResult {