Skip to content

chore: Remove deprecated "azure-container-registry-config" flag for kubelet #306

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 19, 2025
Merged

chore: Remove deprecated "azure-container-registry-config" flag for kubelet #306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8d4afd1
remove deprecated azure-container-registry-config flag
haofan-ms Mar 18, 2025
e4191ed
check flag for update
haofan-ms Mar 18, 2025
e9a8f78
update flag removal
haofan-ms Mar 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions docs/topics/addpool.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ The above assumes that the new node pool specification is in the current working
"kubernetesConfig": {
"kubeletConfig": {
"--cloud-provider": "",
"--cloud-config": "",
"--azure-container-registry-config": ""
"--cloud-config": ""
}
}
}
Expand Down
1 change: 0 additions & 1 deletion docs/topics/clusterdefinitions.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,6 @@ Below is a list of kubelet options that AKS Engine will configure by default:
| "--image-gc-high-threshold" | "85" |
| "--image-gc-low-threshold" | "80" |
| "--non-masquerade-cidr" | "0.0.0.0/0" (unless ip-masq-agent is disabled, in which case this is set to the value of `kubernetesConfig.ClusterSubnet` ) - **Removed in v1.24** |
| "--azure-container-registry-config" | "/etc/kubernetes/azure.json" |
| "--pod-max-pids" | "-1" (need to activate the feature in --feature-gates=SupportPodPidsLimit=true) |
| "--image-pull-progress-deadline" | "30m" |
| "--feature-gates" | No default (can be a comma-separated list). On agent nodes `Accelerators=true` will be applied in the `--feature-gates` option for k8s versions before 1.11.0 |
Expand Down
9 changes: 7 additions & 2 deletions pkg/api/defaults-kubelet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ func (cs *ContainerService) setKubeletConfig(isUpgrade bool) {

// Add Windows-specific overrides
// Eventually paths should not be hardcoded here. They should be relative to $global:KubeDir in the PowerShell script
staticWindowsKubeletConfig["--azure-container-registry-config"] = "c:\\k\\azure.json"
staticWindowsKubeletConfig["--pod-infra-container-image"] = "kubletwin/pause"
staticWindowsKubeletConfig["--kubeconfig"] = "c:\\k\\config"
staticWindowsKubeletConfig["--cloud-config"] = "c:\\k\\azure.json"
Expand Down Expand Up @@ -95,7 +94,6 @@ func (cs *ContainerService) setKubeletConfig(isUpgrade bool) {
"--non-masquerade-cidr": DefaultNonMasqueradeCIDR,
"--cloud-provider": "azure",
"--cloud-config": "/etc/kubernetes/azure.json",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
"--event-qps": DefaultKubeletEventQPS,
"--cadvisor-port": DefaultKubeletCadvisorPort,
"--pod-max-pids": strconv.Itoa(DefaultKubeletPodMaxPIDs),
Expand Down Expand Up @@ -446,6 +444,13 @@ func removeKubeletFlags(k map[string]string, v string) {
delete(k, key)
}
}

// Get rid of values not supported in v1.30 and up
if common.IsKubernetesVersionGe(v, "1.30.0") {
for _, key := range []string{"--azure-container-registry-config"} {
delete(k, key)
}
}
}

func setMissingKubeletValues(p *KubernetesConfig, d map[string]string) {
Expand Down
47 changes: 8 additions & 39 deletions pkg/api/defaults-kubelet_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ func TestKubeletConfigDefaults(t *testing.T) {
delete(expected, "--register-with-taints")

windowsProfileKubeletConfig := cs.Properties.AgentPoolProfiles[1].KubernetesConfig.KubeletConfig
expected["--azure-container-registry-config"] = "c:\\k\\azure.json"
expected["--pod-infra-container-image"] = "kubletwin/pause"
expected["--kubeconfig"] = "c:\\k\\config"
expected["--cloud-config"] = "c:\\k\\azure.json"
Expand Down Expand Up @@ -172,16 +171,13 @@ func TestKubeletConfigDefaults(t *testing.T) {

cs = CreateMockContainerService("testcluster", "", 3, 2, false)
// TODO test all default overrides
// Removed kubelet --azure-container-registry-config deprecated CLI flag
overrideVal := "/etc/override"
cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = map[string]string{
"--azure-container-registry-config": overrideVal,
}
cs.setKubeletConfig(false)
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
for key, val := range map[string]string{"--azure-container-registry-config": overrideVal} {
if k[key] != val {
t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
key, k[key], val)
for key := range map[string]string{"--azure-container-registry-config": overrideVal} {
if _, ok := k[key]; ok {
t.Fatal("got unexpected (removed) '--azure-container-registry-config' kubelet config value")
}
}

Expand All @@ -206,7 +202,6 @@ func getDefaultLinuxKubeletConfig(cs *ContainerService) map[string]string {
"--anonymous-auth": "false",
"--authorization-mode": "Webhook",
"--authentication-token-webhook": "true",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
"--cadvisor-port": "", // Validate that we delete this key for >= 1.12 clusters
"--cgroups-per-qos": "true",
"--client-ca-file": "/etc/kubernetes/certs/ca.crt",
Expand Down Expand Up @@ -262,7 +257,6 @@ func TestKubeletConfigAzureStackDefaults(t *testing.T) {
"--anonymous-auth": "false",
"--authentication-token-webhook": "true",
"--authorization-mode": "Webhook",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
"--cadvisor-port": "", // Validate that we delete this key for >= 1.12 clusters
"--cgroups-per-qos": "true",
"--client-ca-file": "/etc/kubernetes/certs/ca.crt",
Expand Down Expand Up @@ -321,7 +315,6 @@ func TestKubeletConfigAzureStackDefaults(t *testing.T) {
}

windowsProfileKubeletConfig := cs.Properties.AgentPoolProfiles[1].KubernetesConfig.KubeletConfig
expected["--azure-container-registry-config"] = "c:\\k\\azure.json"
expected["--pod-infra-container-image"] = "kubletwin/pause"
expected["--kubeconfig"] = "c:\\k\\config"
expected["--cloud-config"] = "c:\\k\\azure.json"
Expand Down Expand Up @@ -365,16 +358,13 @@ func TestKubeletConfigAzureStackDefaults(t *testing.T) {

cs = CreateMockContainerService("testcluster", "", 3, 2, false)
// TODO test all default overrides
// Removed kubelet --azure-container-registry-config deprecated CLI flag
overrideVal := "/etc/override"
cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = map[string]string{
"--azure-container-registry-config": overrideVal,
}
cs.setKubeletConfig(false)
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
for key, val := range map[string]string{"--azure-container-registry-config": overrideVal} {
if k[key] != val {
t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
key, k[key], val)
for key := range map[string]string{"--azure-container-registry-config": overrideVal} {
if _, ok := k[key]; ok {
t.Fatal("got unexpected (removed) '--azure-container-registry-config' kubelet config value")
}
}

Expand Down Expand Up @@ -469,26 +459,6 @@ func TestKubeletConfigCloudConfig(t *testing.T) {
}
}

func TestKubeletConfigAzureContainerRegistryConfig(t *testing.T) {
// Test default value and custom value for --azure-container-registry-config
cs := CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
cs.setKubeletConfig(false)
k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--azure-container-registry-config"] != "/etc/kubernetes/azure.json" {
t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s",
k["--azure-container-registry-config"])
}

cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--azure-container-registry-config"] = "custom.json"
cs.setKubeletConfig(false)
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--azure-container-registry-config"] != "custom.json" {
t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s",
k["--azure-container-registry-config"])
}
}

func TestKubeletConfigNetworkPlugin(t *testing.T) {
// Test NetworkPlugin = "kubenet"
cs := CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
Expand Down Expand Up @@ -893,7 +863,6 @@ func TestStaticWindowsConfig(t *testing.T) {

// Add Windows-specific overrides
// Eventually paths should not be hardcoded here. They should be relative to $global:KubeDir in the PowerShell script
expected["--azure-container-registry-config"] = "c:\\k\\azure.json"
expected["--pod-infra-container-image"] = "kubletwin/pause"
expected["--kubeconfig"] = "c:\\k\\config"
expected["--cloud-config"] = "c:\\k\\azure.json"
Expand Down
74 changes: 35 additions & 39 deletions pkg/api/defaults_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,18 +128,17 @@ func TestCertsAlreadyPresent(t *testing.T) {
func TestSetMissingKubeletValues(t *testing.T) {
config := &KubernetesConfig{}
defaultKubeletConfig := map[string]string{
"--network-plugin": "1",
"--pod-infra-container-image": "2",
"--max-pods": "3",
"--eviction-hard": "4",
"--node-status-update-frequency": "5",
"--image-gc-high-threshold": "6",
"--image-gc-low-threshold": "7",
"--non-masquerade-cidr": "8",
"--pod-max-pids": "10",
"--cloud-provider": "azure",
"--cloud-config": "/etc/kubernetes/azure.json",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
"--network-plugin": "1",
"--pod-infra-container-image": "2",
"--max-pods": "3",
"--eviction-hard": "4",
"--node-status-update-frequency": "5",
"--image-gc-high-threshold": "6",
"--image-gc-low-threshold": "7",
"--non-masquerade-cidr": "8",
"--pod-max-pids": "10",
"--cloud-provider": "azure",
"--cloud-config": "/etc/kubernetes/azure.json",
}
setMissingKubeletValues(config, defaultKubeletConfig)
for key, val := range defaultKubeletConfig {
Expand All @@ -156,18 +155,17 @@ func TestSetMissingKubeletValues(t *testing.T) {
},
}
expectedResult := map[string]string{
"--network-plugin": "a",
"--pod-infra-container-image": "b",
"--max-pods": "3",
"--eviction-hard": "4",
"--node-status-update-frequency": "5",
"--image-gc-high-threshold": "6",
"--image-gc-low-threshold": "7",
"--non-masquerade-cidr": "8",
"--pod-max-pids": "10",
"--cloud-provider": "",
"--cloud-config": "/etc/kubernetes/azure.json",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
"--network-plugin": "a",
"--pod-infra-container-image": "b",
"--max-pods": "3",
"--eviction-hard": "4",
"--node-status-update-frequency": "5",
"--image-gc-high-threshold": "6",
"--image-gc-low-threshold": "7",
"--non-masquerade-cidr": "8",
"--pod-max-pids": "10",
"--cloud-provider": "",
"--cloud-config": "/etc/kubernetes/azure.json",
}
setMissingKubeletValues(config, defaultKubeletConfig)
for key, val := range expectedResult {
Expand All @@ -178,24 +176,22 @@ func TestSetMissingKubeletValues(t *testing.T) {

config = &KubernetesConfig{
KubeletConfig: map[string]string{
"--cloud-provider": "",
"--cloud-config": "",
"--azure-container-registry-config": "",
"--cloud-provider": "",
"--cloud-config": "",
},
}
expectedResult = map[string]string{
"--network-plugin": "1",
"--pod-infra-container-image": "2",
"--max-pods": "3",
"--eviction-hard": "4",
"--node-status-update-frequency": "5",
"--image-gc-high-threshold": "6",
"--image-gc-low-threshold": "7",
"--non-masquerade-cidr": "8",
"--pod-max-pids": "10",
"--cloud-provider": "",
"--cloud-config": "",
"--azure-container-registry-config": "",
"--network-plugin": "1",
"--pod-infra-container-image": "2",
"--max-pods": "3",
"--eviction-hard": "4",
"--node-status-update-frequency": "5",
"--image-gc-high-threshold": "6",
"--image-gc-low-threshold": "7",
"--non-masquerade-cidr": "8",
"--pod-max-pids": "10",
"--cloud-provider": "",
"--cloud-config": "",
}
setMissingKubeletValues(config, defaultKubeletConfig)
for key, val := range expectedResult {
Expand Down
7 changes: 0 additions & 7 deletions pkg/api/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1613,13 +1613,6 @@ func (a *AgentPoolProfile) RequiresCloudproviderConfig() bool {
} else {
return true
}
if v, ok := a.KubernetesConfig.KubeletConfig["--azure-container-registry-config"]; ok {
if v != "" {
return true
}
} else {
return true
}
} else {
return true
}
Expand Down
30 changes: 7 additions & 23 deletions pkg/api/types_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,8 @@ func TestRequiresCloudproviderConfig(t *testing.T) {
ap: AgentPoolProfile{
KubernetesConfig: &KubernetesConfig{
KubeletConfig: map[string]string{
"--cloud-provider": "azure",
"--cloud-config": "",
"--azure-container-registry-config": "",
"--cloud-provider": "azure",
"--cloud-config": "",
},
},
},
Expand All @@ -506,35 +505,20 @@ func TestRequiresCloudproviderConfig(t *testing.T) {
ap: AgentPoolProfile{
KubernetesConfig: &KubernetesConfig{
KubeletConfig: map[string]string{
"--cloud-provider": "",
"--cloud-config": "/etc/kubernetes/azure.json",
"--azure-container-registry-config": "",
"--cloud-provider": "",
"--cloud-config": "/etc/kubernetes/azure.json",
},
},
},
expected: true,
},
{
name: "--azure-container-registry-config provided",
name: "all flags set explicitly to empty string",
ap: AgentPoolProfile{
KubernetesConfig: &KubernetesConfig{
KubeletConfig: map[string]string{
"--cloud-provider": "",
"--cloud-config": "",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
},
},
},
expected: true,
},
{
name: "all 3 flags set explicitly to empty string",
ap: AgentPoolProfile{
KubernetesConfig: &KubernetesConfig{
KubeletConfig: map[string]string{
"--cloud-provider": "",
"--cloud-config": "",
"--azure-container-registry-config": "",
"--cloud-provider": "",
"--cloud-config": "",
},
},
},
Expand Down
Loading