Skip to content

Commit 33b54e9

Browse files
mythimythi
committed
sgx: add new special resources for TDX QGS and SGX platform registration
Signed-off-by: Mikko Ylinen <[email protected]>
1 parent 769e1d5 commit 33b54e9

File tree

13 files changed

+249
-11
lines changed

13 files changed

+249
-11
lines changed

‎.github/workflows/lib-build.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ jobs:
3333
- accel-config-demo
3434
- intel-opencl-icd
3535
- openssl-qat-engine
36+
- sgx-dcap-infra
3637
- sgx-sdk-demo
3738
- sgx-aesmd-demo
3839
- dsa-dpdk-dmadevtest

‎cmd/sgx_plugin/sgx_plugin.go‎

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
2626
"k8s.io/klog/v2"
2727
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
28+
cdispec "tags.cncf.io/container-device-interface/specs-go"
2829
)
2930

3031
const (
@@ -38,18 +39,20 @@ const (
3839
)
3940

4041
type devicePlugin struct {
41-
scanDone chan bool
42-
devfsDir string
43-
nEnclave uint
44-
nProvision uint
42+
scanDone chan bool
43+
devfsDir string
44+
nEnclave uint
45+
nProvision uint
46+
dcapInfraResources bool
4547
}
4648

47-
func newDevicePlugin(devfsDir string, nEnclave, nProvision uint) *devicePlugin {
49+
func newDevicePlugin(devfsDir string, nEnclave, nProvision uint, dcapInfraResources bool) *devicePlugin {
4850
return &devicePlugin{
49-
devfsDir: devfsDir,
50-
nEnclave: nEnclave,
51-
nProvision: nProvision,
52-
scanDone: make(chan bool, 1),
51+
devfsDir: devfsDir,
52+
nEnclave: nEnclave,
53+
nProvision: nProvision,
54+
dcapInfraResources: dcapInfraResources,
55+
scanDone: make(chan bool, 1),
5356
}
5457
}
5558

@@ -96,6 +99,40 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
9699
devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, nodes, nil, nil, nil, nil, nil))
97100
}
98101

102+
if !dp.dcapInfraResources {
103+
return devTree, nil
104+
}
105+
106+
tdQeNodes := []pluginapi.DeviceSpec{
107+
{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"},
108+
{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"},
109+
}
110+
111+
devTree.AddDevice("tdqe", "tdqe-1", dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, tdQeNodes, nil, nil, nil, nil, nil))
112+
113+
regNodes := []pluginapi.DeviceSpec{
114+
{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"},
115+
{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"},
116+
}
117+
118+
// TODO: /sys/firmware is a maskedPath. Test /run/efivars with a patched PCK-ID-Retrieval-Tool.
119+
efiVarFsMount := &cdispec.Spec{
120+
Version: dpapi.CDIVersion,
121+
Kind: dpapi.CDIVendor + "/sgx",
122+
Devices: []cdispec.Device{
123+
{
124+
Name: "efivarfs",
125+
ContainerEdits: cdispec.ContainerEdits{
126+
Mounts: []*cdispec.Mount{
127+
{HostPath: "efivarfs", ContainerPath: "/run/efivars", Type: "efivarfs", Options: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}},
128+
},
129+
},
130+
},
131+
},
132+
}
133+
134+
devTree.AddDevice("registration", "registration-1", dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, regNodes, nil, nil, nil, nil, efiVarFsMount))
135+
99136
return devTree, nil
100137
}
101138

@@ -121,15 +158,18 @@ func getDefaultPodCount(nCPUs uint) uint {
121158
func main() {
122159
var enclaveLimit, provisionLimit uint
123160

161+
var dcapInfraResources bool
162+
124163
podCount := getDefaultPodCount(uint(runtime.NumCPU()))
125164

126165
flag.UintVar(&enclaveLimit, "enclave-limit", podCount, "Number of \"enclave\" resources")
127166
flag.UintVar(&provisionLimit, "provision-limit", podCount, "Number of \"provision\" resources")
167+
flag.BoolVar(&dcapInfraResources, "dcap-infra-resources", false, "add special resources for DCAP infrastructure daemonSet pods")
128168
flag.Parse()
129169

130170
klog.V(4).Infof("SGX device plugin started with %d \"%s/enclave\" resources and %d \"%s/provision\" resources.", enclaveLimit, namespace, provisionLimit, namespace)
131171

132-
plugin := newDevicePlugin(devicePath, enclaveLimit, provisionLimit)
172+
plugin := newDevicePlugin(devicePath, enclaveLimit, provisionLimit, dcapInfraResources)
133173
manager := dpapi.NewManager(namespace, plugin)
134174
manager.Run()
135175
}

‎cmd/sgx_plugin/sgx_plugin_test.go‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,22 @@ func init() {
2727
_ = flag.Set("v", "4") // Enable debug output
2828
}
2929

30+
// Update if new resource types are added.
31+
const dcapInfraResources = 2
32+
3033
// mockNotifier implements Notifier interface.
3134
type mockNotifier struct {
3235
scanDone chan bool
3336
enclaveDevCount int
3437
provisionDevCount int
38+
dcapInfraResCnt int
3539
}
3640

3741
// Notify stops plugin Scan.
3842
func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) {
3943
n.enclaveDevCount = len(newDeviceTree[deviceTypeEnclave])
4044
n.provisionDevCount = len(newDeviceTree[deviceTypeProvision])
45+
n.dcapInfraResCnt = len(newDeviceTree) - n.enclaveDevCount - n.provisionDevCount
4146
n.scanDone <- true
4247
}
4348

@@ -95,6 +100,7 @@ func TestScan(t *testing.T) {
95100
requestedProvisionDevs uint
96101
expectedEnclaveDevs int
97102
expectedProvisionDevs int
103+
requestDcapInfra bool
98104
}{
99105
{
100106
name: "no device installed",
@@ -131,6 +137,16 @@ func TestScan(t *testing.T) {
131137
requestedProvisionDevs: 20,
132138
expectedProvisionDevs: 20,
133139
},
140+
{
141+
name: "all resources",
142+
enclaveDevice: "sgx_enclave",
143+
provisionDevice: "sgx_provision",
144+
requestedEnclaveDevs: 1,
145+
expectedEnclaveDevs: 1,
146+
requestedProvisionDevs: 1,
147+
expectedProvisionDevs: 1,
148+
requestDcapInfra: true,
149+
},
134150
}
135151

136152
for _, tc := range tcases {
@@ -159,7 +175,7 @@ func TestScan(t *testing.T) {
159175
}
160176
}
161177

162-
plugin := newDevicePlugin(devfs, tc.requestedEnclaveDevs, tc.requestedProvisionDevs)
178+
plugin := newDevicePlugin(devfs, tc.requestedEnclaveDevs, tc.requestedProvisionDevs, tc.requestDcapInfra)
163179

164180
notifier := &mockNotifier{
165181
scanDone: plugin.scanDone,
@@ -175,6 +191,9 @@ func TestScan(t *testing.T) {
175191
if tc.expectedProvisionDevs != notifier.provisionDevCount {
176192
t.Errorf("Wrong number of discovered provision devices")
177193
}
194+
if tc.requestDcapInfra && notifier.dcapInfraResCnt != dcapInfraResources {
195+
t.Errorf("Wrong number of discovered DCAP infra resources: expected %d, got %d.", dcapInfraResources, notifier.dcapInfraResCnt)
196+
}
178197
})
179198
}
180199
}

‎demo/sgx-dcap-infra/Dockerfile‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
FROM ubuntu:24.04 AS builder
2+
3+
RUN apt-get update && \
4+
env DEBIAN_FRONTEND=noninteractive apt-get install -y \
5+
build-essential \
6+
curl \
7+
libcurl4-openssl-dev
8+
9+
WORKDIR /opt/intel
10+
11+
ARG SGX_SDK_URL=https://download.01.org/intel-sgx/sgx-linux/2.26/distro/ubuntu24.04-server/sgx_linux_x64_sdk_2.26.100.0.bin
12+
13+
RUN curl -sSLfO ${SGX_SDK_URL} \
14+
&& export SGX_SDK_INSTALLER=$(basename $SGX_SDK_URL) \
15+
&& chmod +x $SGX_SDK_INSTALLER \
16+
&& echo "yes" | ./$SGX_SDK_INSTALLER \
17+
&& rm $SGX_SDK_INSTALLER
18+
19+
ARG DCAP_VERSION=DCAP_1.23
20+
ARG DCAP_TARBALL_SHA256="c4567e7bc0a2f0dbb70fa2625a9af492e00b96e83d07fa69b9f4f304a9992495"
21+
22+
RUN curl -sSLfO https://github.com/intel/SGXDataCenterAttestationPrimitives/archive/$DCAP_VERSION.tar.gz && \
23+
echo "$DCAP_TARBALL_SHA256 $DCAP_VERSION.tar.gz" | sha256sum -c - && \
24+
tar xzf $DCAP_VERSION.tar.gz && mv SGXDataCenterAttestationPrimitives* SGXDataCenterAttestationPrimitives
25+
26+
WORKDIR SGXDataCenterAttestationPrimitives/tools/PCKRetrievalTool
27+
28+
RUN sed -e 's:sys/firmware/efi:run:g' -i App/utility.cpp \
29+
&& make
30+
31+
FROM ubuntu:24.04
32+
33+
WORKDIR /opt/intel/sgx-pck-id-retrieval-tool/
34+
COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/tools/PCKRetrievalTool/PCKIDRetrievalTool .
35+
36+
RUN ln -sf /lib/x86_64-linux-gnu/libsgx_id_enclave.signed.so.1 && \
37+
ln -sf /lib/x86_64-linux-gnu/libsgx_pce.signed.so.1
38+
39+
ARG SGX_SDK_VERSION=2_26_100
40+
RUN apt update && apt install -y curl gnupg \
41+
&& echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu noble main" | \
42+
tee -a /etc/apt/sources.list.d/intel-sgx.list \
43+
&& curl -s https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | \
44+
gpg --dearmor --output /usr/share/keyrings/intel-sgx.gpg \
45+
&& curl -sFLf https://download.01.org/intel-sgx/sgx_repo/ubuntu/apt_preference_files/99sgx_${SGXSDK_VERSION}_noble_custom_version.cfg | \
46+
tee -a /etc/apt/preferences.d/99sgx_sdk \
47+
&& apt update \
48+
&& apt install -y --no-install-recommends \
49+
libcurl4 \
50+
tdx-qgs \
51+
libsgx-ae-pce \
52+
libsgx-ae-id-enclave \
53+
libsgx-ra-uefi \
54+
libsgx-dcap-default-qpl
55+
56+
# BUG: "qgs -p=0" gets overriden by the config file making the parameter useless
57+
RUN sed -e 's/\(^port =\).*/\1 0/g' -i /etc/qgs.conf
58+
59+
COPY dcap-registration-flow /usr/bin
60+
61+
ENTRYPOINT ["/opt/intel/tdx-qgs/qgs", "--no-daemon", "-p=0"]

‎demo/sgx-dcap-infra/dcap-registration-flow‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
set -u
4+
5+
# TODO remove before merging
6+
sleep infinity
7+
8+
if [ ! -x "${PWD}"/PCKIDRetrievalTool ]; then
9+
echo "dcap-registration-flow: PCKIDRetrievalTool must be in the workingDir and executable"
10+
exit 1
11+
fi
12+
13+
echo "Waiting for the PCCS to be ready ..."
14+
15+
if ! curl --retry 20 --retry-delay 30 -k https://pccs-service:8042/sgx/certification/v4/rootcacrl &> /dev/null; then
16+
echo "ERROR: PCCS pod didn't become ready after 20 minutes"
17+
exit 1
18+
fi
19+
20+
echo "PCCS is online, proceeding ..."
21+
22+
ARGS="-user_token ${USER_TOKEN} -url ${PCCS_URL} -use_secure_cert ${SECURE_CERT}"
23+
24+
echo "Calling PCKIDRetrievalTool ${ARGS} ..."
25+
26+
./PCKIDRetrievalTool ${ARGS}
27+
28+
sleep infinity

‎deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ spec:
5555
spec:
5656
description: SgxDevicePluginSpec defines the desired state of SgxDevicePlugin.
5757
properties:
58+
dcapInfraResources:
59+
description: DcapInfraResources adds two special resources for DCAP
60+
infra DaemonSet Pods.
61+
type: boolean
5862
enclaveLimit:
5963
description: EnclaveLimit is a number of containers that can share
6064
the same SGX enclave device.

‎deployments/sgx_dcap/base/intel-sgx-dcap.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: intel-sgx-dcap-infra
5+
labels:
6+
app: intel-sgx-dcap-infra
7+
spec:
8+
selector:
9+
matchLabels:
10+
app: intel-sgx-dcap-infra
11+
template:
12+
metadata:
13+
labels:
14+
app: intel-sgx-dcap-infra
15+
spec:
16+
automountServiceAccountToken: false
17+
initContainers:
18+
- name: platform-registration
19+
image: intel/sgx-dcap-infra:devel
20+
restartPolicy: Always
21+
workingDir: "/opt/intel/sgx-pck-id-retrieval-tool/"
22+
command: ['/usr/bin/dcap-registration-flow']
23+
securityContext:
24+
readOnlyRootFilesystem: true
25+
allowPrivilegeEscalation: false
26+
capabilities:
27+
drop:
28+
- ALL
29+
resources:
30+
limits:
31+
sgx.intel.com/registration: 1
32+
containers:
33+
- name: tdxqgs
34+
image: intel/sgx-dcap-infra:devel
35+
securityContext:
36+
readOnlyRootFilesystem: true
37+
allowPrivilegeEscalation: false
38+
capabilities:
39+
drop:
40+
- ALL
41+
resources:
42+
limits:
43+
sgx.intel.com/tdqe: 1
44+
imagePullPolicy: IfNotPresent
45+
volumeMounts:
46+
- name: qgs-socket
47+
mountPath: /var/run/tdx-qgs
48+
volumes:
49+
- name: qgs-socket
50+
hostPath:
51+
path: /var/run/tdx-qgs
52+
type: DirectoryOrCreate

‎deployments/sgx_dcap/base/kustomization.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
resources:
2+
- intel-sgx-dcap.yaml
3+
generatorOptions:
4+
disableNameSuffixHash: true

‎deployments/sgx_dcap/kustomization.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
resources:
2+
- base

‎deployments/sgx_plugin/base/intel-sgx-plugin.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ spec:
4949
- name: sgx-provision
5050
mountPath: /dev/sgx_provision
5151
readOnly: true
52+
- name: cdipath
53+
mountPath: /var/run/cdi
5254
volumes:
5355
- name: kubeletsockets
5456
hostPath:
@@ -61,5 +63,9 @@ spec:
6163
hostPath:
6264
path: /dev/sgx_provision
6365
type: CharDevice
66+
- name: cdipath
67+
hostPath:
68+
path: /var/run/cdi
69+
type: DirectoryOrCreate
6470
nodeSelector:
6571
kubernetes.io/arch: amd64

0 commit comments

Comments
 (0)