E2e: Use pre-build node images

lentzi90 · lentzi90 · commit 4fb08c7fb6f6 · 2023-10-11T12:57:48.000+03:00
We are currently using a plain ubuntu cloud image for the nodes in most
e2e tests. Cloud-init scripts are used to install the Kubernetes
components on startup. This is convenient since we do not have to build
specific images. However, the scripts are not well maintained, they
increase the complexity and running time of the tests.

This changes the tests to use a pre-built image. The image was build
using image-builder, which is a standard for many CAPI providers.
diff --git a/Makefile b/Makefile
@@ -156,22 +156,20 @@ E2E_NO_ARTIFACT_TEMPLATES_DIR=test/e2e/data/infrastructure-openstack-no-artifact
 
 .PHONY: e2e-templates
 e2e-templates: ## Generate cluster templates for e2e tests
-e2e-templates: $(addprefix $(E2E_TEMPLATES_DIR)/, \
+e2e-templates: $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \
 		 cluster-template-v1alpha5.yaml \
 		 cluster-template-v1alpha6.yaml \
 		 cluster-template-md-remediation.yaml \
 		 cluster-template-kcp-remediation.yaml \
 		 cluster-template-multi-az.yaml \
 		 cluster-template-multi-network.yaml \
 		 cluster-template-without-lb.yaml \
-		 cluster-template.yaml) \
-		 $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \
+		 cluster-template.yaml \
 		 cluster-template-flatcar.yaml)
+# Currently no templates that require CI artifacts
+# $(addprefix $(E2E_TEMPLATES_DIR)/, add-templates-here.yaml) \
 
-$(E2E_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUSTOMIZE) FORCE
-	$(KUSTOMIZE) build "$<" > "$@"
-
-$(E2E_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE
+$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUSTOMIZE) FORCE
 	$(KUSTOMIZE) build "$<" > "$@"
 
 $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE
diff --git a/hack/ci/cloud-init/controller.yaml.tpl b/hack/ci/cloud-init/controller.yaml.tpl
@@ -53,8 +53,17 @@
 
     # Don't download default images, just our test images
     DOWNLOAD_DEFAULT_IMAGES=False
+    # Increase the total image size limit
+    GLANCE_LIMIT_IMAGE_SIZE_TOTAL=20000
     # We upload the Amphora image so it doesn't have to be build
-    IMAGE_URLS="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/amphora/2022-12-05/amphora-x64-haproxy.qcow2"
+    # Upload the images so we don't have to upload them from Prow
+    # NOTE: If you get issues when changing/adding images, check if the limits
+    # are sufficient and change the variable above if needed.
+    # https://docs.openstack.org/glance/latest/admin/quotas.html
+    IMAGE_URLS="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/amphora/2022-12-05/amphora-x64-haproxy.qcow2,"
+    IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img,"
+    IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.27.2.img,"
+    IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.27.2.img"
 
     [[post-config|$NOVA_CONF]]
     [DEFAULT]
@@ -133,13 +142,6 @@
 
     source /opt/stack/devstack/openrc admin admin
 
-    # Upload the images so we don't have to upload them from Prow
-    # Upload cirros image first in order to avoid reach limit of project
-    # https://docs.openstack.org/glance/latest/admin/quotas.html
-    /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img
-    /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-01-14/focal-server-cloudimg-amd64.img
-    /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.27.2.img
-
     # Add the controller to its own host aggregate and availability zone
     aggregateid=$(openstack aggregate create --zone "${PRIMARY_AZ}" "${PRIMARY_AZ}" -f value -c id)
     for host in $(openstack compute service list --service nova-compute -f value -c Host)
diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml
@@ -117,7 +117,7 @@ providers:
     contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
     replacements:
     - old: "imagePullPolicy: Always"
       new: "imagePullPolicy: IfNotPresent"
@@ -131,7 +131,7 @@ providers:
     contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
     replacements:
     - old: "imagePullPolicy: Always"
       new: "imagePullPolicy: IfNotPresent"
@@ -146,8 +146,8 @@ providers:
     # contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template-without-lb.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template-without-lb.yaml"
     replacements:
     - old: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:dev
       new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
@@ -165,10 +165,6 @@ variables:
   # used to ensure we deploy to the correct management cluster
   KUBE_CONTEXT: "kind-capo-e2e"
   KUBERNETES_VERSION: "v1.27.2"
-  KUBERNETES_VERSION_UPGRADE_FROM: "v1.26.5"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.27.2"
-  ETCD_VERSION_UPGRADE_TO: "3.5.6-0"
-  COREDNS_VERSION_UPGRADE_TO: "v1.9.3"
   CNI: "../../data/cni/calico.yaml"
   CCM: "../../data/ccm/cloud-controller-manager.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
@@ -182,7 +178,7 @@ variables:
   OPENSTACK_DNS_NAMESERVERS: "8.8.8.8"
   OPENSTACK_FAILURE_DOMAIN: "testaz1"
   OPENSTACK_FAILURE_DOMAIN_ALT: "testaz2"
-  OPENSTACK_IMAGE_NAME: "focal-server-cloudimg-amd64"
+  OPENSTACK_IMAGE_NAME: "ubuntu-2204-kube-v1.27.2"
   OPENSTACK_NODE_MACHINE_FLAVOR: "m1.small"
   OPENSTACK_SSH_KEY_NAME: "cluster-api-provider-openstack-sigs-k8s-io"
   OPENSTACK_EXTERNAL_NETWORK_ID: ""
diff --git a/test/e2e/data/kustomize/common-patches/containerd/containerd-kcp.yaml b/test/e2e/data/kustomize/common-patches/containerd/containerd-kcp.yaml
diff --git a/test/e2e/data/kustomize/common-patches/containerd/containerd-kct.yaml b/test/e2e/data/kustomize/common-patches/containerd/containerd-kct.yaml
diff --git a/test/e2e/data/kustomize/common-patches/containerd/kustomization.yaml b/test/e2e/data/kustomize/common-patches/containerd/kustomization.yaml
diff --git a/test/e2e/data/kustomize/default/kustomization.yaml b/test/e2e/data/kustomize/default/kustomization.yaml
@@ -4,6 +4,5 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../upgrade-patches
 - ../common-patches/ccm
diff --git a/test/e2e/data/kustomize/multi-az/patch-machine-template-control-plane.yaml b/test/e2e/data/kustomize/multi-az/patch-machine-template-control-plane.yaml
@@ -2,7 +2,7 @@
 - op: add
   path: /spec/template/spec/rootVolume
   value:
-    diskSize: 15
+    diskSize: 25
 - op: add
   path: /spec/template/spec/additionalBlockDevices
   value:
diff --git a/test/e2e/data/kustomize/multi-az/patch-machine-template-worker.yaml b/test/e2e/data/kustomize/multi-az/patch-machine-template-worker.yaml
@@ -2,7 +2,7 @@
 - op: add
   path: /spec/template/spec/rootVolume
   value:
-    diskSize: 15
+    diskSize: 25
     volumeType: ${OPENSTACK_VOLUME_TYPE_ALT}
     availabilityZone: ${OPENSTACK_FAILURE_DOMAIN}
 - op: add
diff --git a/test/e2e/data/kustomize/v1alpha5/kustomization.yaml b/test/e2e/data/kustomize/v1alpha5/kustomization.yaml
@@ -4,5 +4,4 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../common-patches/ccm
diff --git a/test/e2e/data/kustomize/v1alpha6/kustomization.yaml b/test/e2e/data/kustomize/v1alpha6/kustomization.yaml
@@ -4,7 +4,6 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../common-patches/ccm
 
 patches:
diff --git a/test/e2e/data/kustomize/without-lb/kustomization.yaml b/test/e2e/data/kustomize/without-lb/kustomization.yaml
@@ -4,5 +4,4 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../common-patches/ccm
diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go
@@ -44,14 +44,14 @@ const (
 	OpenStackImageName         = "OPENSTACK_IMAGE_NAME"
 	OpenStackNodeMachineFlavor = "OPENSTACK_NODE_MACHINE_FLAVOR"
 	SSHUserMachine             = "SSH_USER_MACHINE"
-	FlavorDefault              = "ci-artifacts"
-	FlavorWithoutLB            = "without-lb-ci-artifacts"
-	FlavorMultiNetwork         = "multi-network-ci-artifacts"
-	FlavorMultiAZ              = "multi-az-ci-artifacts"
-	FlavorV1alpha5             = "v1alpha5-ci-artifacts"
-	FlavorV1alpha6             = "v1alpha6-ci-artifacts"
-	FlavorMDRemediation        = "md-remediation-ci-artifacts"
-	FlavorKCPRemediation       = "kcp-remediation-ci-artifacts"
+	FlavorDefault              = ""
+	FlavorWithoutLB            = "without-lb"
+	FlavorMultiNetwork         = "multi-network"
+	FlavorMultiAZ              = "multi-az"
+	FlavorV1alpha5             = "v1alpha5"
+	FlavorV1alpha6             = "v1alpha6"
+	FlavorMDRemediation        = "md-remediation"
+	FlavorKCPRemediation       = "kcp-remediation"
 	FlavorFlatcar              = "flatcar"
 )
 
diff --git a/test/e2e/shared/suite.go b/test/e2e/shared/suite.go
@@ -58,6 +58,8 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 	flag.Parse()
 	Expect(e2eCtx.Settings.ConfigPath).To(BeAnExistingFile(), "Invalid test suite argument. configPath should be an existing file.")
 	Expect(os.MkdirAll(e2eCtx.Settings.ArtifactFolder, 0o750)).To(Succeed(), "Invalid test suite argument. Can't create artifacts-folder %q", e2eCtx.Settings.ArtifactFolder)
+	templatesDir := path.Join(e2eCtx.Settings.ArtifactFolder, "templates")
+	Expect(os.MkdirAll(templatesDir, 0o750)).To(Succeed(), "Can't create templates folder %q", templatesDir)
 	Logf("Loading the e2e test configuration from %q", e2eCtx.Settings.ConfigPath)
 	e2eCtx.E2EConfig = LoadE2EConfig(e2eCtx.Settings.ConfigPath)
 
@@ -66,7 +68,8 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 
 	templates := []clusterctl.Files{}
 
-	// TODO(sbuerin): we always need ci artifacts, because we don't have images for every Kubernetes version
+	// Cluster templates in this folder will get ci artifacts injected. It makes it possible to use generic cloud images
+	// without kubernetes pre-installed.
 	err := filepath.WalkDir(path.Join(e2eCtx.Settings.DataFolder, "infrastructure-openstack"), func(f string, d fs.DirEntry, _ error) error {
 		filename := filepath.Base(f)
 		fileExtension := filepath.Ext(filename)
@@ -90,7 +93,7 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 		Expect(err).NotTo(HaveOccurred())
 
 		targetName := fmt.Sprintf("%s-ci-artifacts.yaml", strings.TrimSuffix(filename, fileExtension))
-		targetTemplate := path.Join(e2eCtx.Settings.ArtifactFolder, "templates", targetName)
+		targetTemplate := path.Join(templatesDir, targetName)
 
 		// We have to copy the file from ciTemplate to targetTemplate. Otherwise it would be overwritten because
 		// ciTemplate is the same for all templates
@@ -120,7 +123,7 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 		t, err := os.ReadFile(f)
 		Expect(err).NotTo(HaveOccurred())
 
-		targetTemplate := path.Join(e2eCtx.Settings.ArtifactFolder, "templates", filename)
+		targetTemplate := path.Join(templatesDir, filename)
 
 		err = os.WriteFile(targetTemplate, t, 0o600)
 		Expect(err).NotTo(HaveOccurred())
diff --git a/test/e2e/suites/e2e/e2e_test.go b/test/e2e/suites/e2e/e2e_test.go
@@ -81,7 +81,7 @@ var _ = Describe("e2e tests [PR-Blocking]", func() {
 		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
 		Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
 		Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion))
-		shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false)
+		shared.SetEnvVar("USE_CI_ARTIFACTS", "false", false)
 		postClusterCleanup = nil
 	})
 
@@ -600,7 +600,7 @@ var _ = Describe("e2e tests [PR-Blocking]", func() {
 				rootVolumes[machine.Name] = rootVolume
 				Expect(*rootVolume).To(MatchFields(IgnoreExtras, Fields{
 					"Name":     Equal(fmt.Sprintf("%s-root", server.Name)),
-					"Size":     Equal(15),
+					"Size":     Equal(25),
 					"Bootable": Equal("true"), // This is genuinely a string, not a bool
 				}), "Boot volume %s for machine %s not as expected", rootVolume.ID, machine.Name)
 
