chore: support Kubernetes v1.29.8

cmd/generate_test.go

@@ -553,13 +553,13 @@ func TestExampleAPIModels(t *testing.T) {
 			setArgs:      defaultSet,
 		},
 		{
-			name:         "1.27 example",
-			apiModelPath: "../examples/kubernetes-releases/kubernetes1.27.json",
+			name:         "1.28 example",
+			apiModelPath: "../examples/kubernetes-releases/kubernetes1.28.json",
 			setArgs:      defaultSet,
 		},
 		{
-			name:         "1.28 example",
-			apiModelPath: "../examples/kubernetes-releases/kubernetes1.28.json",
+			name:         "1.29 example",
+			apiModelPath: "../examples/kubernetes-releases/kubernetes1.29.json",
 			setArgs:      defaultSet,
 		},
 		{

examples/kubernetes-releases/kubernetes1.29.json

@@ -0,0 +1,30 @@
+{
+    "apiVersion": "vlabs",
+    "properties": {
+      "orchestratorProfile": {
+        "orchestratorRelease": "1.29"
+      },
+      "masterProfile": {
+        "count": 1,
+        "dnsPrefix": "",
+        "vmSize": "Standard_D2_v3"
+      },
+      "agentPoolProfiles": [
+        {
+          "name": "agentpool1",
+          "count": 3,
+          "vmSize": "Standard_D2_v3"
+        }
+      ],
+      "linuxProfile": {
+        "adminUsername": "azureuser",
+        "ssh": {
+          "publicKeys": [
+            {
+              "keyData": ""
+            }
+          ]
+        }
+      }
+    }
+  }

pkg/api/common/const.go

@@ -57,13 +57,13 @@ const (
 
 const (
 	// KubernetesDefaultRelease is the default Kubernetes release
-	KubernetesDefaultRelease string = "1.27"
+	KubernetesDefaultRelease string = "1.28"
 	// KubernetesDefaultReleaseWindows is the default Kubernetes release for Windows
-	KubernetesDefaultReleaseWindows string = "1.27"
+	KubernetesDefaultReleaseWindows string = "1.28"
 	// KubernetesDefaultReleaseAzureStack is the default Kubernetes release on Azure Stack
-	KubernetesDefaultReleaseAzureStack string = "1.27"
+	KubernetesDefaultReleaseAzureStack string = "1.28"
 	// KubernetesDefaultReleaseWindowsAzureStack is the default Kubernetes release for Windows on Azure Stack
-	KubernetesDefaultReleaseWindowsAzureStack string = "1.27"
+	KubernetesDefaultReleaseWindowsAzureStack string = "1.28"
 	// PodSecurityPolicyRemovedVersion is the first Kubernetes version that does not includes the PSP admission plugin
 	PodSecurityPolicyRemovedVersion = "1.25.0"
 )

pkg/api/common/versions.go

@@ -361,10 +361,11 @@ var AllKubernetesSupportedVersions = map[string]bool{
 	"1.26.9":         false,
 	"1.27.6":         false,
 	"1.27.9":         false,
-	"1.27.10":        true,
+	"1.27.10":        false,
 	"1.28.5":         false,
 	"1.28.6":         false,
 	"1.28.13":        true,
+	"1.29.8":         true,
 }
 
 // AllKubernetesSupportedVersionsAzureStack is a hash table of all supported Kubernetes version strings on Azure Stack
@@ -416,10 +417,11 @@ var AllKubernetesSupportedVersionsAzureStack = map[string]bool{
 	"1.26.9":  false,
 	"1.27.6":  false,
 	"1.27.9":  false,
-	"1.27.10": true,
+	"1.27.10": false,
 	"1.28.5":  false,
 	"1.28.6":  false,
 	"1.28.13": true,
+	"1.29.8":  true,
 }
 
 // AllKubernetesWindowsSupportedVersionsAzureStack maintain a set of available k8s Windows versions in aks-engine on Azure Stack
@@ -466,10 +468,11 @@ var AllKubernetesWindowsSupportedVersionsAzureStack = map[string]bool{
 	"1.26.9":  false,
 	"1.27.6":  false,
 	"1.27.9":  false,
-	"1.27.10": true,
+	"1.27.10": false,
 	"1.28.5":  false,
 	"1.28.6":  false,
 	"1.28.13": true,
+	"1.29.8":  true,
 }
 
 // GetDefaultKubernetesVersion returns the default Kubernetes version, that is the latest patch of the default release

pkg/api/defaults-apiserver.go

@@ -267,6 +267,47 @@ func (cs *ContainerService) overrideAPIServerConfig() {
 		// Reference: https://github.com/kubernetes/kubernetes/pull/117570
 		invalidFeatureGates = append(invalidFeatureGates, "WindowsHostProcessContainers")
 	}
+	if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.29.0") {
+		// Remove --feature-gate CSIMigrationvSphere starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121291
+		invalidFeatureGates = append(invalidFeatureGates, "CSIMigrationvSphere")
+
+		// Remove --feature-gate ProbeTerminationGracePeriod starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121257
+		invalidFeatureGates = append(invalidFeatureGates, "ProbeTerminationGracePeriod")
+
+		// Remove --feature-gate JobTrackingWithFinalizers starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/119100
+		invalidFeatureGates = append(invalidFeatureGates, "JobTrackingWithFinalizers")
+
+		// Remove --feature-gate TopologyManager starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121252
+		invalidFeatureGates = append(invalidFeatureGates, "TopologyManager")
+
+		// Remove --feature-gate OpenAPIV3 starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121255
+		invalidFeatureGates = append(invalidFeatureGates, "OpenAPIV3")
+
+		// Remove --feature-gate SeccompDefault starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121246
+		invalidFeatureGates = append(invalidFeatureGates, "SeccompDefault")
+
+		// Remove --feature-gate CronJobTimeZone, JobMutableNodeSchedulingDirectives, LegacyServiceAccountTokenNoAutoGeneration starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120192
+		invalidFeatureGates = append(invalidFeatureGates, "CronJobTimeZone", "JobMutableNodeSchedulingDirectives", "LegacyServiceAccountTokenNoAutoGeneration")
+
+		// Remove --feature-gate DownwardAPIHugePages starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120249
+		invalidFeatureGates = append(invalidFeatureGates, "DownwardAPIHugePages")
+
+		// Remove --feature-gate GRPCContainerProbe starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120248
+		invalidFeatureGates = append(invalidFeatureGates, "GRPCContainerProbe")
+
+		// Remove --feature-gate RetroactiveDefaultStorageClass starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120861
+		invalidFeatureGates = append(invalidFeatureGates, "RetroactiveDefaultStorageClass")
+	}
 	removeInvalidFeatureGates(o.KubernetesConfig.APIServerConfig, invalidFeatureGates)
 
 	if common.ShouldDisablePodSecurityPolicyAddon(o.OrchestratorVersion) {

pkg/api/defaults-apiserver_test.go

@@ -581,7 +581,7 @@ func TestAPIServerFeatureGates(t *testing.T) {
 	cs := CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
 	cs.setAPIServerConfig()
 	a := cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig
-	if a["--feature-gates"] != "PodSecurity=true" {
+	if a["--feature-gates"] != "" {
 		t.Fatalf("got unexpected '--feature-gates' API server config value for k8s v%s: %s",
 			defaultTestClusterVer, a["--feature-gates"])
 	}
@@ -700,6 +700,33 @@ func TestAPIServerFeatureGates(t *testing.T) {
 		t.Fatalf("got unexpected '--feature-gates' for %s \n API server config original value  %s \n, expected sanitized value: %s \n, actual sanitized value: %s \n ",
 			"1.27.0", featuregate128, a["--feature-gates"], featuregate127Sanitized)
 	}
+
+	// test user-overrides, removal of feature gates for k8s versions >= 1.29
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.29.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig = make(map[string]string)
+	a = cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig
+	featuregate129 := "CSIMigrationvSphere=true,CronJobTimeZone=true,DownwardAPIHugePages=true,GRPCContainerProbe=true,JobMutableNodeSchedulingDirectives=true,JobTrackingWithFinalizers=true,LegacyServiceAccountTokenNoAutoGeneration=true,OpenAPIV3=true,ProbeTerminationGracePeriod=true,RetroactiveDefaultStorageClass=true,SeccompDefault=true,TopologyManager=true"
+	a["--feature-gates"] = featuregate129
+	featuregate129Sanitized := ""
+	cs.setAPIServerConfig()
+	if a["--feature-gates"] != featuregate129Sanitized {
+		t.Fatalf("got unexpected '--feature-gates' for %s \n API server config original value  %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ",
+			"1.29.0", featuregate129, a["--feature-gates"], featuregate129Sanitized)
+	}
+
+	// test user-overrides, no removal of feature gates for k8s versions < 1.29
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.28.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig = make(map[string]string)
+	a = cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig
+	a["--feature-gates"] = featuregate129
+	featuregate128Sanitized = featuregate129
+	cs.setAPIServerConfig()
+	if a["--feature-gates"] != featuregate128Sanitized {
+		t.Fatalf("got unexpected '--feature-gates' for %s \n API server config original value  %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ",
+			"1.28.0", featuregate129, a["--feature-gates"], featuregate128Sanitized)
+	}
 }
 
 func TestAPIServerInsecureFlag(t *testing.T) {

pkg/api/defaults-cloud-controller-manager.go

@@ -115,6 +115,47 @@ func (cs *ContainerService) setCloudControllerManagerConfig() {
 		// Reference: https://github.com/kubernetes/kubernetes/pull/117570
 		invalidFeatureGates = append(invalidFeatureGates, "WindowsHostProcessContainers")
 	}
+	if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.29.0") {
+		// Remove --feature-gate CSIMigrationvSphere starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121291
+		invalidFeatureGates = append(invalidFeatureGates, "CSIMigrationvSphere")
+
+		// Remove --feature-gate ProbeTerminationGracePeriod starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121257
+		invalidFeatureGates = append(invalidFeatureGates, "ProbeTerminationGracePeriod")
+
+		// Remove --feature-gate JobTrackingWithFinalizers starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/119100
+		invalidFeatureGates = append(invalidFeatureGates, "JobTrackingWithFinalizers")
+
+		// Remove --feature-gate TopologyManager starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121252
+		invalidFeatureGates = append(invalidFeatureGates, "TopologyManager")
+
+		// Remove --feature-gate OpenAPIV3 starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121255
+		invalidFeatureGates = append(invalidFeatureGates, "OpenAPIV3")
+
+		// Remove --feature-gate SeccompDefault starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121246
+		invalidFeatureGates = append(invalidFeatureGates, "SeccompDefault")
+
+		// Remove --feature-gate CronJobTimeZone, JobMutableNodeSchedulingDirectives, LegacyServiceAccountTokenNoAutoGeneration starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120192
+		invalidFeatureGates = append(invalidFeatureGates, "CronJobTimeZone", "JobMutableNodeSchedulingDirectives", "LegacyServiceAccountTokenNoAutoGeneration")
+
+		// Remove --feature-gate DownwardAPIHugePages starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120249
+		invalidFeatureGates = append(invalidFeatureGates, "DownwardAPIHugePages")
+
+		// Remove --feature-gate GRPCContainerProbe starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120248
+		invalidFeatureGates = append(invalidFeatureGates, "GRPCContainerProbe")
+
+		// Remove --feature-gate RetroactiveDefaultStorageClass starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120861
+		invalidFeatureGates = append(invalidFeatureGates, "RetroactiveDefaultStorageClass")
+	}
 
 	removeInvalidFeatureGates(o.KubernetesConfig.CloudControllerManagerConfig, invalidFeatureGates)
 

pkg/api/defaults-cloud-controller-manager_test.go

@@ -124,4 +124,31 @@ func TestCloudControllerManagerFeatureGates(t *testing.T) {
 		t.Fatalf("got unexpected '--feature-gates' for %s \n controller manager config original value  %s \n, expected sanitized value: %s \n, actual sanitized value: %s \n ",
 			"1.27.0", featuregate128, ccm["--feature-gates"], featuregate127Sanitized)
 	}
+
+	// test user-overrides, removal of feature gates for k8s versions >= 1.29
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.29.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig = make(map[string]string)
+	ccm = cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig
+	featuregate129 := "CSIMigrationvSphere=true,CronJobTimeZone=true,DownwardAPIHugePages=true,GRPCContainerProbe=true,JobMutableNodeSchedulingDirectives=true,JobTrackingWithFinalizers=true,LegacyServiceAccountTokenNoAutoGeneration=true,OpenAPIV3=true,ProbeTerminationGracePeriod=true,RetroactiveDefaultStorageClass=true,SeccompDefault=true,TopologyManager=true"
+	ccm["--feature-gates"] = featuregate129
+	featuregate129Sanitized := ""
+	cs.setCloudControllerManagerConfig()
+	if ccm["--feature-gates"] != featuregate129Sanitized {
+		t.Fatalf("got unexpected '--feature-gates' for %s \n Cloud Controller Manager config original value  %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ",
+			"1.29.0", featuregate129, ccm["--feature-gates"], featuregate129Sanitized)
+	}
+
+	// test user-overrides, no removal of feature gates for k8s versions < 1.29
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.28.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig = make(map[string]string)
+	ccm = cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig
+	ccm["--feature-gates"] = featuregate129
+	featuregate128Sanitized = featuregate129
+	cs.setCloudControllerManagerConfig()
+	if ccm["--feature-gates"] != featuregate128Sanitized {
+		t.Fatalf("got unexpected '--feature-gates' for %s \n Cloud Controller Manager config original value  %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ",
+			"1.28.0", featuregate129, ccm["--feature-gates"], featuregate128Sanitized)
+	}
 }

pkg/api/defaults-controller-manager.go

@@ -189,5 +189,46 @@ func (cs *ContainerService) setControllerManagerConfig() {
 		// Reference: https://github.com/kubernetes/kubernetes/pull/117570
 		invalidFeatureGates = append(invalidFeatureGates, "WindowsHostProcessContainers")
 	}
+	if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.29.0") {
+		// Remove --feature-gate CSIMigrationvSphere starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121291
+		invalidFeatureGates = append(invalidFeatureGates, "CSIMigrationvSphere")
+
+		// Remove --feature-gate ProbeTerminationGracePeriod starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121257
+		invalidFeatureGates = append(invalidFeatureGates, "ProbeTerminationGracePeriod")
+
+		// Remove --feature-gate JobTrackingWithFinalizers starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/119100
+		invalidFeatureGates = append(invalidFeatureGates, "JobTrackingWithFinalizers")
+
+		// Remove --feature-gate TopologyManager starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121252
+		invalidFeatureGates = append(invalidFeatureGates, "TopologyManager")
+
+		// Remove --feature-gate OpenAPIV3 starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121255
+		invalidFeatureGates = append(invalidFeatureGates, "OpenAPIV3")
+
+		// Remove --feature-gate SeccompDefault starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/121246
+		invalidFeatureGates = append(invalidFeatureGates, "SeccompDefault")
+
+		// Remove --feature-gate CronJobTimeZone, JobMutableNodeSchedulingDirectives, LegacyServiceAccountTokenNoAutoGeneration starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120192
+		invalidFeatureGates = append(invalidFeatureGates, "CronJobTimeZone", "JobMutableNodeSchedulingDirectives", "LegacyServiceAccountTokenNoAutoGeneration")
+
+		// Remove --feature-gate DownwardAPIHugePages starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120249
+		invalidFeatureGates = append(invalidFeatureGates, "DownwardAPIHugePages")
+
+		// Remove --feature-gate GRPCContainerProbe starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120248
+		invalidFeatureGates = append(invalidFeatureGates, "GRPCContainerProbe")
+
+		// Remove --feature-gate RetroactiveDefaultStorageClass starting with 1.29
+		// Reference: https://github.com/kubernetes/kubernetes/pull/120861
+		invalidFeatureGates = append(invalidFeatureGates, "RetroactiveDefaultStorageClass")
+	}
 	removeInvalidFeatureGates(o.KubernetesConfig.ControllerManagerConfig, invalidFeatureGates)
 }