setup-envtest: allow downloading envtest binaries from controller-tools

releases

Author: sbueringer

tools/setup-envtest/README.md

@@ -2,7 +2,8 @@
 
 This is a small tool that manages binaries for envtest. It can be used to
 download new binaries, list currently installed and available ones, and
-clean up versions.
+clean up versions. Binaries can be downloaded either via HTTP via an index
+or from GCS.
 
 To use it, just go-install it on 1.19+ (it's a separate, self-contained
 module):
@@ -40,6 +41,16 @@ setup-envtest use -i --use-env
 
 # sideload a pre-downloaded tarball as Kubernetes 1.16.2 into our store
 setup-envtest sideload 1.16.2 < downloaded-envtest.tar.gz
+
+# Per default envtest binaries are downloaded from: 
+# https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml
+# To download from a custom index use the following:
+setup-envtest use --index https://custom.com/envtest-releases.yaml
+
+# To download from the kubebuilder-tools GCS bucket: (default behavior before v0.18)
+# Note: This is a Google-owned bucket and it might be shutdown at any time
+# see: https://github.com/kubernetes/k8s.io/issues/2647#event-12439345373
+setup-envtest use --use-gcs
 ```
 
 ## Where does it put all those binaries?
@@ -51,16 +62,16 @@ On Linux, this is `$XDG_DATA_HOME`; on Windows, `%LocalAppData`; and on
 OSX, `~/Library/Application Support`.
 
 There's an overall folder that holds all files, and inside that is
-a folder for each version/platform pair.  The exact directory structure is
-not guarnateed, except that the leaf directory will contain the names
-expected by envtest.  You should always use `setup-envtest fetch` or
+a folder for each version/platform pair. The exact directory structure is
+not guaranteed, except that the leaf directory will contain the names
+expected by envtest. You should always use `setup-envtest fetch` or
 `setup-envtest switch` (generally with the `-p path` or `-p env` flags) to
 get the directory that you should use.
 
 ## Why do I have to do that `source <(blah blah blah)` thing
 
 This is a normal binary, not a shell script, so we can't set the parent
-process's environment variables.  If you use this by hand a lot and want
+process's environment variables. If you use this by hand a lot and want
 to save the typing, you could put something like the following in your
 `~/.zshrc` (or similar for bash/fish/whatever, modified to those):
 
@@ -79,7 +90,7 @@ setup-envtest() {
 There are a few options.
 
 First, you'll probably want to set the `-i/--installed` flag. If you want
-to avoid forgetting to set this flag, set  the `ENVTEST_INSTALLED_ONLY`
+to avoid forgetting to set this flag, set the `ENVTEST_INSTALLED_ONLY`
 env variable, which will switch that flag on by default.
 
 Then, you have a few options for managing your binaries:
@@ -98,13 +109,17 @@ Then, you have a few options for managing your binaries:
   `--use-env` on by default.
 
 - If you want to use this tool, but download your gziped tarballs
-  separately, you can use the `sideload` command.  You'll need to use the
+  separately, you can use the `sideload` command. You'll need to use the
   `-k/--version` flag to indicate which version you're sideloading.
 
   After that, it'll be as if you'd installed the binaries with `use`.
 
-- If you want to talk to some internal source, you can use the
-  `--remote-bucket` and `--remote-server` options.  The former sets which
+- If you want to talk to some internal source via HTTP, you can simply set `--index`
+  The index must contain references to envtest binary archives in the same format as:
+  https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml
+
+- If you want to talk to some internal source in a GCS "style", you can use the
+  `--remote-bucket` and `--remote-server` options together with `--use-gcs`. The former sets which
   GCS bucket to download from, and the latter sets the host to talk to as
   if it were a GCS endpoint. Theoretically, you could use the latter
   version to run an internal "mirror" -- the tool expects
@@ -114,7 +129,7 @@ Then, you have a few options for managing your binaries:
     ```json
     {"items": [
         {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": "<base-64-encoded-md5-hash>"},
-        {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": "<base-64-encoded-md5-hash>"},
+        {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": "<base-64-encoded-md5-hash>"}
     ]}
     ```
 

tools/setup-envtest/env/env.go

@@ -33,17 +33,21 @@ type Env struct {
 	// Platform is our current platform
 	Platform versions.PlatformItem
 
-	// VerifiySum indicates whether or not we should run checksums.
+	// VerifySum indicates whether we should run checksums.
 	VerifySum bool
-	// NoDownload forces us to not contact GCS, looking only
-	// at local files instead.
+	// NoDownload forces us to not contact GCS or download the index via HTTP,
+	// looking only at local files instead.
 	NoDownload bool
 	// ForceDownload forces us to ignore local files and always
-	// contact GCS & re-download.
+	// contact GCS or download the index via HTTP & re-download.
 	ForceDownload bool
 
-	// Client is our remote client for contacting GCS.
-	Client *remote.Client
+	// UseGCS signals if the GCS client is used.
+	UseGCS bool
+
+	// Client is our remote client for contacting GCS or
+	// to download the index via HTTP.
+	Client remote.Client
 
 	// Log allows us to log.
 	Log logr.Logger
@@ -133,7 +137,7 @@ func (e *Env) ListVersions(ctx context.Context) {
 }
 
 // LatestVersion returns the latest version matching our version selector and
-// platform from the remote server, with the correspoding checksum for later
+// platform from the remote server, with the corresponding checksum for later
 // use as well.
 func (e *Env) LatestVersion(ctx context.Context) (versions.Concrete, versions.PlatformItem) {
 	vers, err := e.Client.ListVersions(ctx)
@@ -193,7 +197,7 @@ func (e *Env) ExistsAndValid() bool {
 //
 // If necessary, it will enumerate on-disk and remote versions to accomplish
 // this, finding a version that matches our version selector and platform.
-// It will always yield a concrete version, it *may* yield a concrete platorm
+// It will always yield a concrete version, it *may* yield a concrete platform
 // as well.
 func (e *Env) EnsureVersionIsSet(ctx context.Context) {
 	if e.Version.AsConcrete() != nil {
@@ -247,13 +251,13 @@ func (e *Env) EnsureVersionIsSet(ctx context.Context) {
 
 	// if we're not forcing a download, and we have a newer local version, just use that
 	if !e.ForceDownload && localVer != nil && localVer.NewerThan(serverVer) {
-		e.Platform.Platform = localPlat // update our data with md5
+		e.Platform.Platform = localPlat // update our data with hash
 		e.Version.MakeConcrete(*localVer)
 		return
 	}
 
 	// otherwise, use the new version from the server
-	e.Platform = platform // update our data with md5
+	e.Platform = platform // update our data with hash
 	e.Version.MakeConcrete(serverVer)
 }
 
@@ -266,13 +270,13 @@ func (e *Env) Fetch(ctx context.Context) {
 	log := e.Log.WithName("fetch")
 
 	// if we didn't just fetch it, grab the sum to verify
-	if e.VerifySum && e.Platform.MD5 == "" {
+	if e.VerifySum && e.Platform.Hash == nil {
 		if err := e.Client.FetchSum(ctx, *e.Version.AsConcrete(), &e.Platform); err != nil {
-			ExitCause(2, err, "unable to fetch checksum for requested version")
+			ExitCause(2, err, "unable to fetch hash for requested version")
 		}
 	}
 	if !e.VerifySum {
-		e.Platform.MD5 = "" // skip verification
+		e.Platform.Hash = nil // skip verification
 	}
 
 	var packedPath string
@@ -287,7 +291,7 @@ func (e *Env) Fetch(ctx context.Context) {
 		}
 	})
 
-	archiveOut, err := e.FS.TempFile("", "*-"+e.Platform.ArchiveName(*e.Version.AsConcrete()))
+	archiveOut, err := e.FS.TempFile("", "*-"+e.Platform.ArchiveName(e.UseGCS, *e.Version.AsConcrete()))
 	if err != nil {
 		ExitCause(2, err, "unable to open file to write downloaded archive to")
 	}
@@ -365,8 +369,8 @@ func (e *Env) PrintInfo(printFmt PrintFormat) {
 	case PrintOverview:
 		fmt.Fprintf(e.Out, "Version: %s\n", e.Version)
 		fmt.Fprintf(e.Out, "OS/Arch: %s\n", e.Platform)
-		if e.Platform.MD5 != "" {
-			fmt.Fprintf(e.Out, "md5: %s\n", e.Platform.MD5)
+		if e.Platform.Hash != nil {
+			fmt.Fprintf(e.Out, "%s: %s\n", e.Platform.Hash.Type, e.Platform.Hash.Value)
 		}
 		fmt.Fprintf(e.Out, "Path: %s\n", path)
 	case PrintPath:
@@ -409,7 +413,7 @@ func (e *Env) Sideload(ctx context.Context, input io.Reader) {
 }
 
 var (
-	// expectedExectuables are the executables that are checked in PathMatches
+	// expectedExecutables are the executables that are checked in PathMatches
 	// for non-store paths.
 	expectedExecutables = []string{
 		"kube-apiserver",
@@ -458,7 +462,7 @@ func (e *Env) PathMatches(value string) bool {
 }
 
 // versionFromPathName checks if the given path's last component looks like one
-// of our versions, and, if so, what version it represents.  If succesfull,
+// of our versions, and, if so, what version it represents.  If successful,
 // it'll set version and platform, and return true.  Otherwise it returns
 // false.
 func (e *Env) versionFromPathName(value string) bool {

tools/setup-envtest/go.mod

@@ -3,25 +3,27 @@ module sigs.k8s.io/controller-runtime/tools/setup-envtest
 go 1.22.0
 
 require (
-	github.com/go-logr/logr v1.2.4
-	github.com/go-logr/zapr v1.2.4
-	github.com/onsi/ginkgo/v2 v2.12.1
-	github.com/onsi/gomega v1.27.10
+	github.com/go-logr/logr v1.4.1
+	github.com/go-logr/zapr v1.3.0
+	github.com/onsi/ginkgo/v2 v2.17.1
+	github.com/onsi/gomega v1.32.0
 	github.com/spf13/afero v1.6.0
 	github.com/spf13/pflag v1.0.5
 	go.uber.org/zap v1.26.0
+	k8s.io/apimachinery v0.0.0-20240424173219-03f2f3350dc5
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
-	golang.org/x/tools v0.12.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.18.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )