Merge pull request #1513 from mythi/PR-2023-040

k8s.io 1.28.1 update and SGX rework

Author: hj-johannes-lee

.trivyignore

@@ -9,11 +9,6 @@ AVD-DS-0002
 # initcontainers require privileged access
 AVD-KSV-0017
 
-# Sharing the host’s network namespace permits processes in the pod to communicate with
-# processes bound to the host’s loopback adapter.
-# sgx single-node demo deployment uses hostNetwork: true to be able to use the default setting for PCCS URL from containers
-AVD-KSV-0009
-
 # Do not allow privilege escalation from node proxy
 # Check whether role permits privilege escalation from node proxy
 # gpu plugin in kubelet mode requires "nodes/proxy" resource access

cmd/sgx_plugin/README.md

@@ -195,8 +195,10 @@ Successfully tagged intel/sgx-sdk-demo:devel
 #### Deploy the pods
 
 The demo runs Intel aesmd (architectural enclaves service daemon) that is responsible
-for generating SGX quotes for workloads. It is deployed with `hostNetwork: true`
-to allow connections to localhost PCCS.
+for generating SGX quotes for workloads.
+
+**Note**: The PCCS URL must be configured in `sgx_default_qcnl.conf`. The default `localhost` URL
+is not available in containers
 
 ```bash
 $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_aesmd?ref=<RELEASE_VERSION>'
@@ -239,5 +241,7 @@ $ kubectl logs ecdsa-quote-intelsgx-demo-job-vtq84
   Step4: Call sgx_qe_get_quote:succeed!cert_key_type = 0x5
 ```
 
+Similarly, full SGX DCAP Flow with Quote Generation and Trusted Quote Verification can be deployed using the `sgx_ecdsa_inproc_quote` overlay. Again, the PCCS URL must be set beforehand.
+
 > **Note**: The deployment example above uses [kustomize](https://github.com/kubernetes-sigs/kustomize)
 > that is available in kubectl since Kubernetes v1.14 release.

demo/screencast-sgx.sh

@@ -27,7 +27,7 @@ cleanup()
   out 'Cleanup demo artifacts' 20
   out 'delete node-feature-discovery deployment:' 20
   command 'kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=main || true' 20
-  command 'kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=main || true' 20
+  command 'kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd?ref=main || true' 20
   out 'delete SGX Device Plugin deployment:' 20
   command 'kubectl delete sgxdeviceplugin sgxdeviceplugin-sample || true' 20
   out 'delete Intel Device Plugin Operator deployment:' 20
@@ -69,10 +69,10 @@ screen3()
   clear
   out "2. Deploy node-feature-discovery for Kubernetes"
   out "It's used to label SGX capable nodes and register SGX EPC as an extended resource"
-  command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=main"
+  command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd?ref=main"
   out "Check its pod is running"
   command "kubectl wait --for=condition=Ready pod/$(kubectl get --no-headers -l app=nfd-worker -o=jsonpath='{.items[0].metadata.name}' pods -n node-feature-discovery) -n node-feature-discovery"
-  out "Create NodeFeatureRules for SGX specific labels"
+  out "Create NodeFeatureRules for SGX specific labels and SGX EPC extended resource"
   command 'kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=main || true' 20
 }
 
@@ -91,8 +91,8 @@ screen5()
 {
   clear
   out "4. Verify node resources"
-  command "kubectl get nodes -o json | jq .items[].status.allocatable | grep sgx"
-  command "kubectl get nodes -o json | jq .items[].metadata.labels | grep sgx"
+  command "kubectl get nodes -o jsonpath='{.items[].status.allocatable}' | jq | grep sgx"
+  command "kubectl get nodes -o jsonpath='{.items[].metadata.labels}' | jq | grep kubernetes.io\/sgx"
   out "Both node labels and resources for SGX are in place"
 }
 
@@ -104,7 +104,10 @@ screen6()
   command "sudo ctr -n k8s.io i import sgx-aesmd.tar"
   command "sudo ctr -n k8s.io i import sgx-demo.tar"
   out "Deploy Intel(R) AESMD"
-  command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_aesmd?ref=main -n sgx-ecdsa-quote"
+  pushd ../deployments/sgx_aesmd/base
+  jq --arg pccs_url "$PCCS_URL" '.pccs_url = $pccs_url' sgx_default_qcnl.template > sgx_default_qcnl.conf
+  command "kubectl apply -k . -n sgx-ecdsa-quote"
+  popd
   out "Deploy Intel(R) SGX DCAP ECDSA Quote Generation"
   command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_aesmd_quote?ref=main -n sgx-ecdsa-quote"
   command "kubectl logs $(kubectl get --no-headers -l job-name=ecdsa-quote-intelsgx-demo-job -o=jsonpath='{.items[0].metadata.name}' pods -n sgx-ecdsa-quote) -n sgx-ecdsa-quote"
@@ -117,11 +120,14 @@ screen6()
 screen7()
 {
   clear
-  out "6. Run Intel(R) SGX DCAP ECDSA Quote Generation (in-proc)"
-  out "Deploy Intel(R) SGX DCAP ECDSA Quote Generation"
-  command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote?ref=main -n sgx-ecdsa-quote"
+  out "6. Run Intel(R) SGX DCAP ECDSA Quote Generation (in-proc) and Trusted Quote Verification"
+  out "Deploy Intel(R) SGX DCAP ECDSA DCAP Flow"
+  pushd ../deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote
+  jq --arg pccs_url "$PCCS_URL" '.pccs_url = $pccs_url' sgx_default_qcnl.template > sgx_default_qcnl.conf
+  command "kubectl apply -k . -n sgx-ecdsa-quote"
+  popd
   command "kubectl logs $(kubectl get --no-headers -l job-name=inproc-ecdsa-quote-intelsgx-demo-job -o=jsonpath='{.items[0].metadata.name}' pods -n sgx-ecdsa-quote) -n sgx-ecdsa-quote"
-  out "Intel(R) SGX DCAP QuoteGenerationSample successfully generated a quote using DCAP Quote Provider Library"
+  out "Intel(R) SGX DCAP QuoteGenerationSample successfully generated and verified a quote using DCAP Quote Provider Library"
   out "Delete the deployment"
   command "kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote?ref=main -n sgx-ecdsa-quote"
 }
@@ -134,6 +140,7 @@ screen8()
   out "* SGX Kubernetes* Device Plugin deployment with an Operator"
   out "* Intel(R) SGX node resource and feature label registration to Kubernetes*"
   out "* Intel(R) SGX DCAP ECDSA Quote Generation (out-of-proc and in-proc)"
+  out "* Intel(R) SGX DCAP ECDSA Trusted Quote Verification"
 }
 
 if [ "$1" == 'play' ] ; then

demo/sgx-sdk-demo/Dockerfile

@@ -23,7 +23,7 @@ RUN apt-get update && \
 # SGX SDK is installed in /opt/intel directory.
 WORKDIR /opt/intel
 
-ARG DCAP_VERSION=DCAP_1.17
+ARG DCAP_VERSION=DCAP_1.18
 
 RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main" | \
     tee -a /etc/apt/sources.list.d/intel-sgx.list \
@@ -32,11 +32,12 @@ RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://d
  && apt-get update \
  && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
     libsgx-dcap-ql-dev \
+    libsgx-dcap-quote-verify-dev \
     libsgx-dcap-default-qpl-dev \
     libsgx-quote-ex-dev
 
 # Install SGX SDK
-ARG SGX_SDK_URL=https://download.01.org/intel-sgx/sgx-linux/2.20/distro/ubuntu22.04-server/sgx_linux_x64_sdk_2.20.100.4.bin
+ARG SGX_SDK_URL=https://download.01.org/intel-sgx/sgx-linux/2.21/distro/ubuntu22.04-server/sgx_linux_x64_sdk_2.21.100.1.bin
 RUN wget ${SGX_SDK_URL} \
  && export SGX_SDK_INSTALLER=$(basename $SGX_SDK_URL) \
  && chmod +x $SGX_SDK_INSTALLER \
@@ -55,6 +56,12 @@ RUN cd SGXDataCenterAttestationPrimitives/SampleCode/QuoteGenerationSample \
     && make \
     && cd -
 
+RUN cd SGXDataCenterAttestationPrimitives/SampleCode/QuoteVerificationSample \
+    && . /opt/intel/sgxsdk/environment \
+    && make HW_RELEASE=1 \
+    && sgx_sign sign -key ../QuoteGenerationSample/Enclave/Enclave_private_sample.pem -enclave enclave.so -out enclave.signed.so -config Enclave/Enclave.config.xml \
+    && cd -
+
 FROM ubuntu:22.04
 
 RUN apt-get update && \
@@ -72,9 +79,12 @@ RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://d
     libsgx-enclave-common \
     libsgx-urts \
     libsgx-quote-ex \
+    libsgx-dcap-quote-verify \
+    libsgx-ae-qve \
     libsgx-dcap-ql \
     libsgx-dcap-default-qpl \
  && mkdir -p /opt/intel/sgx-sample-app/ \
+ && mkdir -p /opt/intel/sgx-quote-verification/ \
  && mkdir -p /opt/intel/sgx-quote-generation/
 
 COPY --from=builder /opt/intel/sgxsdk/SampleCode/SampleEnclave/app /opt/intel/sgx-sample-app/sgx-sample-app
@@ -83,4 +93,9 @@ COPY --from=builder /opt/intel/sgxsdk/SampleCode/SampleEnclave/enclave.signed.so
 COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteGenerationSample/app /opt/intel/sgx-quote-generation/sgx-quote-generation
 COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteGenerationSample/enclave.signed.so /opt/intel/sgx-quote-generation/enclave.signed.so
 
+COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteVerificationSample/app /opt/intel/sgx-quote-verification/sgx-quote-verification
+COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteVerificationSample/enclave.signed.so /opt/intel/sgx-quote-verification/enclave.signed.so
+
+COPY --chmod=555 run-dcap-flow /opt/intel
+
 ENTRYPOINT /opt/intel/sgx-sample-app/sgx-sample-app

demo/sgx-sdk-demo/run-dcap-flow

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+pushd sgx-quote-generation
+
+./sgx-quote-generation
+
+popd
+
+pushd sgx-quote-verification
+
+./sgx-quote-verification -quote ../sgx-quote-generation/quote.dat

deployments/sgx_aesmd/base/intel-sgx-aesmd.yaml

@@ -15,7 +15,6 @@ spec:
       annotations:
         sgx.intel.com/quote-provider: "aesmd"
     spec:
-      hostNetwork: true
       containers:
       - name: aesmd
         image: intel/sgx-aesmd-demo:devel

deployments/sgx_aesmd/base/sgx_default_qcnl.template

@@ -0,0 +1,5 @@
+{
+  "pccs_url": "https://localhost:8081/sgx/certification/v4/",
+  "use_secure_cert": false,
+  "pccs_api_version": "3.1"
+}

deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/add_hostnetwork.yaml

@@ -0,0 +1,5 @@
+[
+  {"op": "replace", "path": "/spec/template/spec/containers/0/workingDir", "value": "/opt/intel/"},
+  {"op": "replace", "path": "/spec/template/spec/containers/0/command", "value": ["/opt/intel/run-dcap-flow"]},
+  {"op": "remove", "path": "/spec/template/spec/containers/0/securityContext/readOnlyRootFilesystem"}
+]

deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/change_workingdir_and_command.json

@@ -10,5 +10,10 @@ configMapGenerator:
   - sgx_default_qcnl.conf
   name: sgx-attestation-conf
 patches:
-- path: add_hostnetwork.yaml
 - path: add_sgx_default_qcnl_conf.yaml
+- path: change_workingdir_and_command.json
+  target:
+    group: batch
+    kind: Job
+    name: intelsgx-demo-job
+    version: v1