✨ Add e2e profiling toolchain for heap and CPU analysis

Introduces automated memory and CPU profiling for e2e tests with:
- Automatic port-forwarding to Kubernetes deployment pprof endpoints
- Configurable periodic heap and CPU profile collection with differential timing
- Analysis report generation with growth metrics and top allocators
- Makefile targets: start-profiling, stop-profiling, analyze-profiles

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>
Signed-off-by: Todd Short <[email protected]>

Author: tmshort

.gitignore

@@ -50,3 +50,6 @@ site
 
 # Temporary files and directories
 /test/regression/convert/testdata/tmp/*
+
+# Test profiling artifacts
+test-profiles/

Makefile

@@ -106,11 +106,11 @@ CATALOGS_MANIFEST := $(MANIFEST_HOME)/default-catalogs.yaml
 
 .PHONY: help
 help: #HELP Display essential help.
-	@awk 'BEGIN {FS = ":[^#]*#HELP"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\n"} /^[a-zA-Z_0-9-]+:.*#HELP / { printf "  \033[36m%-21s\033[0m %s\n", $$1, $$2 } ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":[^#]*#HELP"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\n"} /^[a-zA-Z_0-9\/%-]+:.*#HELP / { printf "  \033[36m%-21s\033[0m %s\n", $$1, $$2 } ' $(MAKEFILE_LIST)
 
 .PHONY: help-extended
 help-extended: #HELP Display extended help.
-	@awk 'BEGIN {FS = ":.*#(EX)?HELP"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*#(EX)?HELP / { printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^#SECTION / { printf "\n\033[1m%s\033[0m\n", substr($$0, 10) } ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*#(EX)?HELP"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9\/%-]+:.*#(EX)?HELP / { printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^#SECTION / { printf "\n\033[1m%s\033[0m\n", substr($$0, 10) } ' $(MAKEFILE_LIST)
 
 #SECTION Development
 
@@ -335,6 +335,27 @@ test-upgrade-experimental-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade e2e t
 e2e-coverage:
 	COVERAGE_NAME=$(COVERAGE_NAME) ./hack/test/e2e-coverage.sh
 
+TEST_PROFILE_BIN := bin/test-profile
+.PHONY: build-test-profiler
+build-test-profiler: #EXHELP Build the test profiling tool
+	cd hack/tools/test-profiling && go build -o ../../../$(TEST_PROFILE_BIN) ./cmd/test-profile
+
+.PHONY: test-test-profiler
+test-test-profiler: #EXHELP Run unit tests for the test profiling tool
+	cd hack/tools/test-profiling && go test -v ./...
+
+.PHONY: start-profiling
+start-profiling: build-test-profiler #EXHELP Start profiling in background with auto-generated name (timestamp). Use start-profiling/<name> for custom name.
+	$(TEST_PROFILE_BIN) start
+
+.PHONY: start-profiling/%
+start-profiling/%: build-test-profiler #EXHELP Start profiling in background with specified name. Usage: make start-profiling/<name>
+	$(TEST_PROFILE_BIN) start $*
+
+.PHONY: stop-profiling
+stop-profiling: build-test-profiler #EXHELP Stop profiling and generate analysis report
+	$(TEST_PROFILE_BIN) stop
+
 #SECTION KIND Cluster Operations
 
 .PHONY: kind-load

hack/tools/test-profiling/README.md

@@ -0,0 +1,95 @@
+# Test Profiling Tools
+
+Collect and analyze heap/CPU profiles during operator-controller tests.
+
+## Quick Start
+
+```bash
+# Start profiling
+make start-profiling/baseline
+
+# Run tests
+make test-e2e
+
+# Stop and analyze
+make stop-profiling
+
+# View report
+cat test-profiles/baseline/analysis.md
+
+# Compare runs
+./bin/test-profile compare baseline optimized
+cat test-profiles/comparisons/baseline-vs-optimized.md
+```
+
+## Commands
+
+```bash
+# Build
+make build-test-profiler
+
+# Run test with profiling
+./bin/test-profile run <name> [test-target]
+
+# Start/stop daemon
+./bin/test-profile start [name]  # Daemonizes automatically
+./bin/test-profile stop
+
+# Analyze/compare
+./bin/test-profile analyze <name>
+./bin/test-profile compare <baseline> <optimized>
+./bin/test-profile collect  # Single snapshot
+```
+
+## Configuration
+
+```bash
+# Define components to profile (optional - defaults to operator-controller and catalogd)
+# Format: "name:namespace:deployment:port;name2:namespace2:deployment2:port2"
+export TEST_PROFILE_COMPONENTS="operator-controller:olmv1-system:operator-controller-controller-manager:6060;catalogd:olmv1-system:catalogd-controller-manager:6060"
+
+# Profile custom applications
+export TEST_PROFILE_COMPONENTS="my-app:my-ns:my-deployment:8080;api-server:api-ns:api-deployment:9090"
+
+# Other settings
+export TEST_PROFILE_INTERVAL=10                # seconds between collections
+export TEST_PROFILE_CPU_DURATION=10            # CPU profiling duration in seconds
+export TEST_PROFILE_MODE=both                  # both|heap|cpu
+export TEST_PROFILE_DIR=./test-profiles        # output directory
+export TEST_PROFILE_TEST_TARGET=test-e2e       # make target to run
+```
+
+**Component Configuration:**
+- Each component needs a `/debug/pprof` endpoint (standard Go pprof)
+- Local ports are automatically assigned to avoid conflicts
+- Default: operator-controller and catalogd in olmv1-system namespace
+
+## Output
+
+```
+test-profiles/
+├── <name>/
+│   ├── operator-controller/{heap,cpu}*.pprof
+│   ├── catalogd/{heap,cpu}*.pprof
+│   ├── profiler.log
+│   └── analysis.md
+└── comparisons/<name>-vs-<name>.md
+```
+
+## Interactive Analysis
+
+```bash
+cd test-profiles/<name>/operator-controller
+go tool pprof -top heap23.pprof
+go tool pprof -base=heap0.pprof -top heap23.pprof
+go tool pprof -text heap23.pprof | grep -i openapi
+```
+
+## Requirements
+
+**Runtime:**
+- Kubernetes cluster access (via kubeconfig)
+
+**Build:**
+- go 1.24+
+- make

hack/tools/test-profiling/cmd/test-profile/analyze.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/operator-framework/operator-controller/hack/tools/test-profiling/pkg/analyzer"
+	"github.com/operator-framework/operator-controller/hack/tools/test-profiling/pkg/config"
+	"github.com/spf13/cobra"
+)
+
+var analyzeCmd = &cobra.Command{
+	Use:   "analyze <name>",
+	Short: "Analyze collected profiles",
+	Long: `Generate an analysis report from previously collected profiles.
+
+The report includes:
+- Memory growth analysis
+- Top memory allocators
+- CPU profiling results
+- OpenAPI and JSON deserialization analysis
+
+Example:
+  test-profile analyze baseline`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAnalyze,
+}
+
+func runAnalyze(cmd *cobra.Command, args []string) error {
+	cfg := config.DefaultConfig()
+	cfg.Name = args[0]
+
+	if err := cfg.Validate(); err != nil {
+		return err
+	}
+
+	fmt.Printf("📊 Analyzing profiles in: %s\n", cfg.ProfileDir())
+
+	a := analyzer.NewAnalyzer(cfg)
+	return a.Analyze()
+}

hack/tools/test-profiling/cmd/test-profile/collect.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	"github.com/operator-framework/operator-controller/hack/tools/test-profiling/pkg/collector"
+	"github.com/operator-framework/operator-controller/hack/tools/test-profiling/pkg/config"
+	"github.com/spf13/cobra"
+)
+
+var collectCmd = &cobra.Command{
+	Use:   "collect",
+	Short: "Collect a single profile snapshot",
+	Long: `Collect a single snapshot of heap and CPU profiles from all components.
+
+This is useful for quick spot checks without running the full daemon.
+
+Example:
+  test-profile collect`,
+	RunE: runCollect,
+}
+
+func runCollect(cmd *cobra.Command, args []string) error {
+	cfg := config.DefaultConfig()
+	cfg.Name = time.Now().Format("snapshot-20060102-150405")
+
+	if err := cfg.Validate(); err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+
+	c := collector.NewCollector(cfg)
+	return c.CollectOnce(ctx)
+}

hack/tools/test-profiling/cmd/test-profile/compare.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/operator-framework/operator-controller/hack/tools/test-profiling/pkg/comparator"
+	"github.com/operator-framework/operator-controller/hack/tools/test-profiling/pkg/config"
+	"github.com/spf13/cobra"
+)
+
+var compareCmd = &cobra.Command{
+	Use:   "compare <baseline> <optimized>",
+	Short: "Compare two profile runs",
+	Long: `Generate a comparison report between two profile runs.
+
+This helps identify improvements or regressions between different
+versions or configurations.
+
+Example:
+  test-profile compare baseline optimized`,
+	Args: cobra.ExactArgs(2),
+	RunE: runCompare,
+}
+
+func runCompare(cmd *cobra.Command, args []string) error {
+	cfg := config.DefaultConfig()
+
+	baselineName := args[0]
+	optimizedName := args[1]
+
+	baselineDir := filepath.Join(cfg.OutputDir, baselineName)
+	optimizedDir := filepath.Join(cfg.OutputDir, optimizedName)
+	outputDir := filepath.Join(cfg.OutputDir, "comparisons")
+
+	fmt.Printf("📊 Comparing %s vs %s\n", baselineName, optimizedName)
+
+	c := comparator.NewComparator(baselineDir, optimizedDir, outputDir)
+	return c.Compare(baselineName, optimizedName)
+}

hack/tools/test-profiling/cmd/test-profile/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"os"
+)
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

hack/tools/test-profiling/cmd/test-profile/root.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "test-profile",
+	Short: "Test profiling tool for operator-controller",
+	Long: `Test profiling tool for collecting, analyzing, and comparing
+heap and CPU profiles during operator-controller tests.
+
+Examples:
+  # Run test with profiling
+  test-profile run baseline
+
+  # Start profiling daemon
+  test-profile start my-test
+
+  # Stop profiling daemon
+  test-profile stop
+
+  # Analyze collected profiles
+  test-profile analyze baseline
+
+  # Compare two runs
+  test-profile compare baseline optimized
+
+  # Collect single snapshot
+  test-profile collect`,
+}
+
+func init() {
+	rootCmd.AddCommand(runCmd)
+	rootCmd.AddCommand(startCmd)
+	rootCmd.AddCommand(stopCmd)
+	rootCmd.AddCommand(collectCmd)
+	rootCmd.AddCommand(analyzeCmd)
+	rootCmd.AddCommand(compareCmd)
+}