Skip to content

etcd-manager: Use image volumes to mount etcd images #17539

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 10, 2025
Merged

etcd-manager: Use image volumes to mount etcd images #17539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3f99d04
etcd-manager: Use image volumes to mount etcd images
hakman Aug 3, 2025
da4af99
hack/update-expected.sh
hakman Aug 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions pkg/apis/kops/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package kops
import (
"fmt"

"github.com/blang/semver/v4"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -947,6 +948,29 @@ func (c *Cluster) InstallCNIAssets() bool {
c.Spec.Networking.Cilium == nil
}

func (c *Cluster) HasImageVolumesSupport() bool {
// Image Volumes was added to Kubernetes v1.31
// https://kubernetes.io/blog/2024/08/16/kubernetes-1-31-image-volume-source/
// Image Volumes graduated to beta in Kubernetes v1.33
// https://kubernetes.io/blog/2025/04/29/kubernetes-v1-33-image-volume-beta/
if c.IsKubernetesLT("1.33.0") {
return false
}
if c.Spec.Containerd == nil || c.Spec.Containerd.Version == nil {
return false
}
sv, err := semver.ParseTolerant(*c.Spec.Containerd.Version)
if err != nil {
return false
}
// Image Volumes was released in Containerd v2.1.0
// https://github.com/containerd/containerd/releases/tag/v2.1.0
if sv.LT(semver.MustParse("2.1.0")) {
return false
}
return true
}

func (c *Cluster) APIInternalName() string {
return "api.internal." + c.ObjectMeta.Name
}
Expand Down
31 changes: 30 additions & 1 deletion pkg/model/components/etcdmanager/model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,22 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
}
}

{
if b.Cluster.HasImageVolumesSupport() {
for _, etcdVersion := range etcdSupportedVersions() {
if etcdVersion.SymlinkToVersion == "" {
volume := v1.Volume{
Name: "etcd-v" + strings.ReplaceAll(etcdVersion.Version, ".", "-"),
VolumeSource: v1.VolumeSource{
Image: &v1.ImageVolumeSource{
Reference: b.AssetBuilder.RemapImage(etcdVersion.Image),
PullPolicy: v1.PullIfNotPresent,
},
},
}
pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
}
}
} else {
utilMounts := []v1.VolumeMount{
{
MountPath: "/opt",
Expand Down Expand Up @@ -331,6 +346,20 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance

// Remap image via AssetBuilder
container.Image = b.AssetBuilder.RemapImage(container.Image)

if b.Cluster.HasImageVolumesSupport() {
for _, etcdVersion := range etcdSupportedVersions() {
volumeMount := v1.VolumeMount{
MountPath: "/opt/etcd-v" + etcdVersion.Version,
}
if etcdVersion.SymlinkToVersion == "" {
volumeMount.Name = "etcd-v" + strings.ReplaceAll(etcdVersion.Version, ".", "-")
} else {
volumeMount.Name = "etcd-v" + strings.ReplaceAll(etcdVersion.SymlinkToVersion, ".", "-")
}
container.VolumeMounts = append(container.VolumeMounts, volumeMount)
}
}
}

var clientHost string
Expand Down
14 changes: 14 additions & 0 deletions pkg/model/components/etcdmanager/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,20 @@ var _ loader.ClusterOptionsBuilder = &EtcdManagerOptionsBuilder{}
func (b *EtcdManagerOptionsBuilder) BuildOptions(o *kops.Cluster) error {
clusterSpec := &o.Spec

// Image Volumes will become GA in Kubernetes 1.35
// https://github.com/kubernetes/enhancements/pull/5450
if b.ControlPlaneKubernetesVersion().IsLT("1.35.0") && o.HasImageVolumesSupport() {
if clusterSpec.ControlPlaneKubelet == nil {
clusterSpec.ControlPlaneKubelet = &kops.KubeletConfigSpec{}
}
if clusterSpec.ControlPlaneKubelet.FeatureGates == nil {
clusterSpec.ControlPlaneKubelet.FeatureGates = make(map[string]string)
}
if _, found := clusterSpec.ControlPlaneKubelet.FeatureGates["ImageVolume"]; !found {
clusterSpec.ControlPlaneKubelet.FeatureGates["ImageVolume"] = "true"
}
}

for i := range clusterSpec.EtcdClusters {
etcdCluster := &clusterSpec.EtcdClusters[i]
if etcdCluster.Backups == nil {
Expand Down
4 changes: 3 additions & 1 deletion pkg/model/components/etcdmanager/tests/minimal/cluster.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ spec:
provider: Manager
backups:
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
kubernetesVersion: v1.21.0
containerd:
version: 2.1.4
kubernetesVersion: v1.33.0
masterPublicName: api.minimal.example.com
networkCIDR: 172.20.0.0/16
networking:
Expand Down
204 changes: 64 additions & 140 deletions pkg/model/components/etcdmanager/tests/minimal/tasks.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,80 +102,34 @@ Contents: |
name: pki
- mountPath: /opt
name: opt
- mountPath: /opt/etcd-v3.4.13
name: etcd-v3-4-13
- mountPath: /opt/etcd-v3.4.3
name: etcd-v3-4-13
- mountPath: /opt/etcd-v3.5.0
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.1
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.13
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.17
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.21
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.3
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.4
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.6
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.7
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.9
name: etcd-v3-5-21
- mountPath: /var/log/etcd.log
name: varlogetcd
hostNetwork: true
hostPID: true
initContainers:
- args:
- --target-dir=/opt/kops-utils/
- --src=/ko-app/kops-utils-cp
command:
- /ko-app/kops-utils-cp
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
name: kops-utils-cp
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --target-dir=/opt/etcd-v3.4.13
- --src=/usr/local/bin/etcd
- --src=/usr/local/bin/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/etcd:v3.4.13
name: init-etcd-3-4-13
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --target-dir=/opt/etcd-v3.5.21
- --src=/usr/local/bin/etcd
- --src=/usr/local/bin/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/etcd:v3.5.21
name: init-etcd-3-5-21
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --symlink
- --target-dir=/opt/etcd-v3.4.3
- --src=/opt/etcd-v3.4.13/etcd
- --src=/opt/etcd-v3.4.13/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
name: init-etcd-symlinks-3-4-13
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --symlink
- --target-dir=/opt/etcd-v3.5.0
- --target-dir=/opt/etcd-v3.5.1
- --target-dir=/opt/etcd-v3.5.13
- --target-dir=/opt/etcd-v3.5.17
- --target-dir=/opt/etcd-v3.5.3
- --target-dir=/opt/etcd-v3.5.4
- --target-dir=/opt/etcd-v3.5.6
- --target-dir=/opt/etcd-v3.5.7
- --target-dir=/opt/etcd-v3.5.9
- --src=/opt/etcd-v3.5.21/etcd
- --src=/opt/etcd-v3.5.21/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
name: init-etcd-symlinks-3-5-21
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
priorityClassName: system-cluster-critical
tolerations:
- key: CriticalAddonsOnly
Expand All @@ -195,6 +149,14 @@ Contents: |
name: pki
- emptyDir: {}
name: opt
- image:
pullPolicy: IfNotPresent
reference: registry.k8s.io/etcd:v3.4.13
name: etcd-v3-4-13
- image:
pullPolicy: IfNotPresent
reference: registry.k8s.io/etcd:v3.5.21
name: etcd-v3-5-21
- hostPath:
path: /var/log/etcd-events.log
type: FileOrCreate
Expand Down Expand Up @@ -244,80 +206,34 @@ Contents: |
name: pki
- mountPath: /opt
name: opt
- mountPath: /opt/etcd-v3.4.13
name: etcd-v3-4-13
- mountPath: /opt/etcd-v3.4.3
name: etcd-v3-4-13
- mountPath: /opt/etcd-v3.5.0
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.1
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.13
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.17
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.21
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.3
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.4
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.6
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.7
name: etcd-v3-5-21
- mountPath: /opt/etcd-v3.5.9
name: etcd-v3-5-21
- mountPath: /var/log/etcd.log
name: varlogetcd
hostNetwork: true
hostPID: true
initContainers:
- args:
- --target-dir=/opt/kops-utils/
- --src=/ko-app/kops-utils-cp
command:
- /ko-app/kops-utils-cp
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
name: kops-utils-cp
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --target-dir=/opt/etcd-v3.4.13
- --src=/usr/local/bin/etcd
- --src=/usr/local/bin/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/etcd:v3.4.13
name: init-etcd-3-4-13
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --target-dir=/opt/etcd-v3.5.21
- --src=/usr/local/bin/etcd
- --src=/usr/local/bin/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/etcd:v3.5.21
name: init-etcd-3-5-21
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --symlink
- --target-dir=/opt/etcd-v3.4.3
- --src=/opt/etcd-v3.4.13/etcd
- --src=/opt/etcd-v3.4.13/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
name: init-etcd-symlinks-3-4-13
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
- args:
- --symlink
- --target-dir=/opt/etcd-v3.5.0
- --target-dir=/opt/etcd-v3.5.1
- --target-dir=/opt/etcd-v3.5.13
- --target-dir=/opt/etcd-v3.5.17
- --target-dir=/opt/etcd-v3.5.3
- --target-dir=/opt/etcd-v3.5.4
- --target-dir=/opt/etcd-v3.5.6
- --target-dir=/opt/etcd-v3.5.7
- --target-dir=/opt/etcd-v3.5.9
- --src=/opt/etcd-v3.5.21/etcd
- --src=/opt/etcd-v3.5.21/etcdctl
command:
- /opt/kops-utils/kops-utils-cp
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
name: init-etcd-symlinks-3-5-21
resources: {}
volumeMounts:
- mountPath: /opt
name: opt
priorityClassName: system-cluster-critical
tolerations:
- key: CriticalAddonsOnly
Expand All @@ -337,6 +253,14 @@ Contents: |
name: pki
- emptyDir: {}
name: opt
- image:
pullPolicy: IfNotPresent
reference: registry.k8s.io/etcd:v3.4.13
name: etcd-v3-4-13
- image:
pullPolicy: IfNotPresent
reference: registry.k8s.io/etcd:v3.5.21
name: etcd-v3-5-21
- hostPath:
path: /var/log/etcd.log
type: FileOrCreate
Expand Down
2 changes: 1 addition & 1 deletion ...mal-1.33/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ ClusterName: minimal.example.com
ConfigBase: memfs://tests/minimal.example.com
InstanceGroupName: master-us-test-1a
InstanceGroupRole: ControlPlane
NodeupConfigHash: xbmJGssuIxsJRK9XhaKClsBg2o8f0AgSGPWoqHTTX7s=
NodeupConfigHash: PtkgcAnStiVeR9dBVfWvjbyz7gMCuGX83Y49gsjcrqU=

__EOF_KUBE_ENV

Expand Down
2 changes: 2 additions & 0 deletions ...integration/update_cluster/minimal-1.33/data/aws_s3_object_cluster-completed.spec_content
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ spec:
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
featureGates:
ImageVolume: "true"
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
podInfraContainerImage: registry.k8s.io/pause:3.9
Expand Down
Loading
Loading