Skip to content

Commit 3e5c218

Browse files
sujeet01sujeet01
committed
Enhance cert-manager integration for metrics endpoints
1 parent 2e4d763 commit 3e5c218

File tree

32 files changed

+909
-314
lines changed

32 files changed

+909
-314
lines changed

‎cmd/ironcore-controller-manager/main.go‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"flag"
99
"fmt"
1010
"os"
11+
"path/filepath"
1112
"time"
1213

1314
corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1"
@@ -29,6 +30,7 @@ import (
2930
quotaevaluatorironcore "github.com/ironcore-dev/ironcore/internal/quota/evaluator/ironcore"
3031
"github.com/ironcore-dev/ironcore/utils/quota"
3132
"k8s.io/utils/lru"
33+
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
3234
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
3335
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3436

@@ -103,6 +105,7 @@ func init() {
103105
func main() {
104106
var metricsAddr string
105107
var secureMetrics bool
108+
var metricsCertPath, metricsCertName, metricsCertKey string
106109
var enableHTTP2 bool
107110
var enableLeaderElection bool
108111
var probeAddr string
@@ -115,8 +118,12 @@ func main() {
115118
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
116119
flag.BoolVar(&secureMetrics, "metrics-secure", true,
117120
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
121+
flag.StringVar(&metricsCertPath, "metrics-cert-path", "",
122+
"The directory that contains the metrics server certificate.")
123+
flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
124+
flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
118125
flag.BoolVar(&enableHTTP2, "enable-http2", false,
119-
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
126+
"If set, HTTP/2 will be enabled for the metrics server")
120127
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
121128
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
122129
"Enable leader election for controller manager. "+
@@ -191,7 +198,7 @@ func main() {
191198
tlsOpts = append(tlsOpts, disableHTTP2)
192199
}
193200

194-
// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
201+
// Metrics endpoint is enabled in 'config/controller/default/kustomization.yaml'. The Metrics options configure the server.
195202
// More info:
196203
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
197204
// - https://book.kubebuilder.io/reference/metrics.html
@@ -207,10 +214,37 @@ func main() {
207214
// can access the metrics endpoint. The RBAC are configured in 'config/controller/rbac/kustomization.yaml'. More info:
208215
// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
209216
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
217+
}
218+
219+
// If the certificate is not specified, controller-runtime will automatically
220+
// generate self-signed certificates for the metrics server. While convenient for development and testing,
221+
// this setup is not recommended for production.
222+
//
223+
// TODO(user): If you enable certManager, uncomment the following lines:
224+
// - [METRICS-WITH-CERTS] at config/controller/default/kustomization.yaml to generate and use certificates
225+
// managed by cert-manager for the metrics server.
226+
// - [PROMETHEUS-WITH-CERTS] at config/controller/prometheus/kustomization.yaml for TLS certification.
227+
228+
// Create watchers for metrics certificates
229+
var metricsCertWatcher *certwatcher.CertWatcher
230+
231+
if len(metricsCertPath) > 0 {
232+
setupLog.Info("Initializing metrics certificate watcher using provided certificates",
233+
"metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey)
234+
235+
var err error
236+
metricsCertWatcher, err = certwatcher.New(
237+
filepath.Join(metricsCertPath, metricsCertName),
238+
filepath.Join(metricsCertPath, metricsCertKey),
239+
)
240+
if err != nil {
241+
setupLog.Error(err, "to initialize metrics certificate watcher", "error", err)
242+
os.Exit(1)
243+
}
210244

211-
// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
212-
// generate self-signed certificates for the metrics server. While convenient for development and testing,
213-
// this setup is not recommended for production.
245+
metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
246+
config.GetCertificate = metricsCertWatcher.GetCertificate
247+
})
214248
}
215249

216250
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{

‎config/bucketpoollet-broker/certmanager/certificate.yaml‎ renamed to ‎config/bucketpoollet-broker/certmanager/certificate-metrics.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ spec:
1212
apiVersion: cert-manager.io/v1
1313
kind: Certificate
1414
metadata:
15-
name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
15+
name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml
1616
namespace: system
1717
spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
1818
dnsNames:
@@ -21,4 +21,4 @@ spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomiz
2121
issuerRef:
2222
kind: Issuer
2323
name: selfsigned-issuer
24-
secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
24+
secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize

‎config/bucketpoollet-broker/certmanager/kustomization.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
resources:
2-
- certificate.yaml
2+
- certificate-metrics.yaml
33

44
configurations:
55
- kustomizeconfig.yaml

‎config/bucketpoollet-broker/default/cert_metrics_manager_patch.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs.
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
name: controller-manager
6+
namespace: system
7+
spec:
8+
template:
9+
spec:
10+
containers:
11+
- name: manager
12+
volumeMounts:
13+
- mountPath: /tmp/k8s-metrics-server/metrics-certs
14+
name: metrics-certs
15+
readOnly: true
16+
args:
17+
- "--metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs"
18+
volumes:
19+
- name: metrics-certs
20+
secret:
21+
secretName: metrics-server-cert
22+
optional: false
23+
items:
24+
- key: ca.crt
25+
path: ca.crt
26+
- key: tls.crt
27+
path: tls.crt
28+
- key: tls.key
29+
path: tls.key

‎config/bucketpoollet-broker/default/kustomization.yaml‎

Lines changed: 139 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -13,101 +13,148 @@ namePrefix: bucketpoollet-
1313
# someName: someValue
1414

1515
resources:
16-
- ../broker-rbac
17-
- ../manager
18-
# - ../webhook
19-
# - ../certmanager
16+
- ../broker-rbac
17+
- ../manager
18+
# - ../webhook
19+
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
20+
# - ../certmanager
2021
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
21-
#- ../prometheus
22+
# - ../prometheus
2223

2324
patchesStrategicMerge:
24-
- manager_metrics_patch.yaml
25+
# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
26+
# More info: https://book.kubebuilder.io/reference/metrics
27+
- manager_metrics_patch.yaml
28+
29+
# Uncomment the patches line if you enable Metrics and CertManager
30+
# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
31+
# This patch will protect the metrics with certManager self-signed certs.
32+
# - cert_metrics_manager_patch.yaml
2533

26-
# Mount the controller config file for loading manager configurations
27-
# through a ComponentConfig type
28-
#- manager_config_patch.yaml
34+
# Mount the controller config file for loading manager configurations
35+
# through a ComponentConfig type
36+
#- manager_config_patch.yaml
2937

30-
#- manager_webhook_patch.yaml
31-
#- webhookcainjection_patch.yaml
38+
#- manager_webhook_patch.yaml
39+
#- webhookcainjection_patch.yaml
3240

33-
#replacements:
34-
# - source:
35-
# kind: Certificate
36-
# group: cert-manager.io
37-
# version: v1
38-
# name: serving-cert
39-
# fieldPath: .metadata.namespace
40-
# targets:
41-
# - select:
42-
# kind: ValidatingWebhookConfiguration
43-
# fieldPaths:
44-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
45-
# options:
46-
# delimiter: '/'
47-
# index: 0
48-
# create: true
49-
# - select:
50-
# kind: MutatingWebhookConfiguration
51-
# fieldPaths:
52-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
53-
# options:
54-
# delimiter: '/'
55-
# index: 0
56-
# create: true
57-
# - source:
58-
# kind: Certificate
59-
# group: cert-manager.io
60-
# version: v1
61-
# name: serving-cert
62-
# fieldPath: .metadata.name
63-
# targets:
64-
# - select:
65-
# kind: ValidatingWebhookConfiguration
66-
# fieldPaths:
67-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
68-
# options:
69-
# delimiter: '/'
70-
# index: 1
71-
# create: true
72-
# - select:
73-
# kind: MutatingWebhookConfiguration
74-
# fieldPaths:
75-
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
76-
# options:
77-
# delimiter: '/'
78-
# index: 1
79-
# create: true
80-
# - source:
81-
# kind: Service
82-
# version: v1
83-
# name: webhook-service
84-
# fieldPath: .metadata.name
85-
# targets:
86-
# - select:
87-
# kind: Certificate
88-
# group: cert-manager.io
89-
# version: v1
90-
# fieldPaths:
91-
# - .spec.dnsNames.0
92-
# - .spec.dnsNames.1
93-
# options:
94-
# delimiter: '.'
95-
# index: 0
96-
# create: true
97-
# - source:
98-
# kind: Service
99-
# version: v1
100-
# name: webhook-service
101-
# fieldPath: .metadata.namespace
102-
# targets:
103-
# - select:
104-
# kind: Certificate
105-
# group: cert-manager.io
106-
# version: v1
107-
# fieldPaths:
108-
# - .spec.dnsNames.0
109-
# - .spec.dnsNames.1
110-
# options:
111-
# delimiter: '.'
112-
# index: 1
113-
# create: true
41+
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
42+
# Uncomment the following replacements to add the cert-manager CA injection annotations
43+
# replacements:
44+
# - source: # Uncomment the following block to enable certificates for metrics
45+
# kind: Service
46+
# version: v1
47+
# name: controller-manager-metrics-service
48+
# fieldPath: metadata.name
49+
# targets:
50+
# - select:
51+
# kind: Certificate
52+
# group: cert-manager.io
53+
# version: v1
54+
# name: metrics-certs
55+
# fieldPaths:
56+
# - spec.dnsNames.0
57+
# - spec.dnsNames.1
58+
# options:
59+
# delimiter: '.'
60+
# index: 0
61+
# create: true
62+
# - source:
63+
# kind: Service
64+
# version: v1
65+
# name: controller-manager-metrics-service
66+
# fieldPath: metadata.namespace
67+
# targets:
68+
# - select:
69+
# kind: Certificate
70+
# group: cert-manager.io
71+
# version: v1
72+
# name: metrics-certs
73+
# fieldPaths:
74+
# - spec.dnsNames.0
75+
# - spec.dnsNames.1
76+
# options:
77+
# delimiter: '.'
78+
# index: 1
79+
# create: true
80+
#
81+
# - source:
82+
# kind: Certificate
83+
# group: cert-manager.io
84+
# version: v1
85+
# name: serving-cert
86+
# fieldPath: .metadata.namespace
87+
# targets:
88+
# - select:
89+
# kind: ValidatingWebhookConfiguration
90+
# fieldPaths:
91+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
92+
# options:
93+
# delimiter: '/'
94+
# index: 0
95+
# create: true
96+
# - select:
97+
# kind: MutatingWebhookConfiguration
98+
# fieldPaths:
99+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
100+
# options:
101+
# delimiter: '/'
102+
# index: 0
103+
# create: true
104+
# - source:
105+
# kind: Certificate
106+
# group: cert-manager.io
107+
# version: v1
108+
# name: serving-cert
109+
# fieldPath: .metadata.name
110+
# targets:
111+
# - select:
112+
# kind: ValidatingWebhookConfiguration
113+
# fieldPaths:
114+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
115+
# options:
116+
# delimiter: '/'
117+
# index: 1
118+
# create: true
119+
# - select:
120+
# kind: MutatingWebhookConfiguration
121+
# fieldPaths:
122+
# - .metadata.annotations.[cert-manager.io/inject-ca-from]
123+
# options:
124+
# delimiter: '/'
125+
# index: 1
126+
# create: true
127+
# - source:
128+
# kind: Service
129+
# version: v1
130+
# name: webhook-service
131+
# fieldPath: .metadata.name
132+
# targets:
133+
# - select:
134+
# kind: Certificate
135+
# group: cert-manager.io
136+
# version: v1
137+
# fieldPaths:
138+
# - .spec.dnsNames.0
139+
# - .spec.dnsNames.1
140+
# options:
141+
# delimiter: '.'
142+
# index: 0
143+
# create: true
144+
# - source:
145+
# kind: Service
146+
# version: v1
147+
# name: webhook-service
148+
# fieldPath: .metadata.namespace
149+
# targets:
150+
# - select:
151+
# kind: Certificate
152+
# group: cert-manager.io
153+
# version: v1
154+
# fieldPaths:
155+
# - .spec.dnsNames.0
156+
# - .spec.dnsNames.1
157+
# options:
158+
# delimiter: '.'
159+
# index: 1
160+
# create: true

‎config/bucketpoollet-broker/prometheus/kustomization.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,11 @@
11
resources:
22
- monitor.yaml
3+
4+
# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
5+
# to securely reference certificates created and managed by cert-manager.
6+
# Additionally, ensure that you uncomment the [METRICS-WITH-CERTS] patch under config/bucketpoollet-broker/default/kustomization.yaml
7+
# to mount the "metrics-server-cert" secret in the Manager Deployment.
8+
# patches:
9+
# - path: monitor_tls_patch.yaml
10+
# target:
11+
# kind: ServiceMonitor

0 commit comments

Comments
 (0)