From e541ebf7e86a438e731a87cb4e5cc6f3acd3cc99 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 13 May 2024 19:49:21 -0400 Subject: [PATCH] ensure PodTemplateSpec has default values filled in Call Scheme.Default() on the PodSpecTemplate before returning it to Kueue to ensure that Kueue's equality comparisons work as expected. --- go.mod | 3 +++ go.sum | 6 ++++++ pkg/utils/utils.go | 24 +++++++++++++++++------- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index a81f34a..814a183 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( k8s.io/api v0.29.2 k8s.io/apimachinery v0.29.2 k8s.io/client-go v0.29.2 + k8s.io/kubernetes v1.29.2 k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/controller-runtime v0.17.0 sigs.k8s.io/kueue v0.6.2 @@ -20,6 +21,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -45,6 +47,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect diff --git a/go.sum b/go.sum index 631a86c..c6cf1b7 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= @@ -92,6 +94,8 @@ github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5n github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -225,6 +229,8 @@ k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs= k8s.io/kube-aggregator v0.28.1/go.mod h1:JaLizMe+AECSpO2OmrWVsvnG0V3dX1RpW+Wq/QHbu18= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kubernetes v1.29.2 h1:8hh1cntqdulanjQt7wSSSsJfBgOyx6fUdFWslvGL5m0= +k8s.io/kubernetes v1.29.2/go.mod h1:xZPKU0yO0CBbLTnbd+XGyRmmtmaVuJykDb8gNCkeeUE= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 4258298..0794bda 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -26,13 +26,21 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + pkgcorev1 "k8s.io/kubernetes/pkg/apis/core/v1" "k8s.io/utils/ptr" workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" ) +var scheme = runtime.NewScheme() + const templateString = "template" +func init() { + utilruntime.Must(pkgcorev1.AddToScheme(scheme)) +} + // GetPodTemplateSpec extracts a Kueue-compatible PodTemplateSpec at the given path within obj func GetPodTemplateSpec(obj *unstructured.Unstructured, path string) (*v1.PodTemplateSpec, error) { candidatePTS, err := GetRawTemplate(obj.UnstructuredContent(), path) @@ -41,27 +49,29 @@ func GetPodTemplateSpec(obj *unstructured.Unstructured, path string) (*v1.PodTem } // Extract the PodSpec that should be at candidatePTS.spec + podTemplate := &v1.PodTemplate{} spec, ok := candidatePTS["spec"].(map[string]interface{}) if !ok { return nil, fmt.Errorf("content at %v does not contain a spec", path) } - podSpec := &v1.PodSpec{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(spec, podSpec, true); err != nil { + if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(spec, &podTemplate.Template.Spec, true); err != nil { return nil, fmt.Errorf("content at %v.spec not parseable as a v1.PodSpec: %w", path, err) } - // Construct the filtered PodTemplateSpec, copying only the metadata expected by Kueue - template := &v1.PodTemplateSpec{Spec: *podSpec} + // Set default values. Required for proper operation of Kueue's ComparePodSetSlices + scheme.Default(podTemplate) + + // Copy in the subset of the metadate expected by Kueye. if metadata, ok := candidatePTS["metadata"].(map[string]interface{}); ok { if labels, ok := metadata["labels"].(map[string]string); ok { - template.ObjectMeta.Labels = labels + podTemplate.Template.ObjectMeta.Labels = labels } if annotations, ok := metadata["annotations"].(map[string]string); ok { - template.ObjectMeta.Annotations = annotations + podTemplate.Template.ObjectMeta.Annotations = annotations } } - return template, nil + return &podTemplate.Template, nil } // GetReplicas parses the value at the given path within obj as an int