Merge pull request #293 from zvigrinberg/feature/rediness-probe-external-services

zvigrinberg · web-flow · commit 6685d0fdfac6 · 2024-03-06T17:03:24.000+02:00
feat: add readiness health check for external services
diff --git a/deploy/exhort.yaml b/deploy/exhort.yaml
@@ -64,8 +64,8 @@ spec:
           httpGet:
             path: /q/health/ready
             port: 9000
-          initialDelaySeconds: 5
-          periodSeconds: 20
+          initialDelaySeconds: 2
+          periodSeconds: 15
 ---
 apiVersion: v1
 kind: Service
diff --git a/deploy/openshift/template.yaml b/deploy/openshift/template.yaml
@@ -77,8 +77,8 @@ objects:
                 httpGet:
                   path: /q/health/ready
                   port: '${{MANAGEMENT_PORT}}'
-                initialDelaySeconds: 5
-                periodSeconds: 10
+                initialDelaySeconds: 2
+                periodSeconds: 15
               ports:
                 - name: http
                   containerPort: '${{SERVICE_PORT}}'
diff --git a/src/main/java/com/redhat/exhort/integration/Constants.java b/src/main/java/com/redhat/exhort/integration/Constants.java
@@ -31,7 +31,12 @@ public final class Constants {
 
   private Constants() {}
 
+  public static final String PROVIDER_NAME = "providerName";
+
+  public static final String EXCLUDE_FROM_READINESS_CHECK = "exclusionFromReadiness";
   public static final String PROVIDERS_PARAM = "providers";
+
+  public static final String HEALTH_CHECKS_LIST_HEADER_NAME = "healthChecksRoutesList";
   public static final String SBOM_TYPE_PARAM = "sbomType";
 
   public static final String ACCEPT_HEADER = "Accept";
@@ -90,8 +95,11 @@ private Constants() {}
   public static final String SNYK_DEP_GRAPH_API_PATH = "/test/dep-graph";
   public static final String SNYK_TOKEN_API_PATH = "/user/me";
   public static final String OSS_INDEX_AUTH_COMPONENT_API_PATH = "/authorized/component-report";
+  public static final String OSS_INDEX_VERSION_PATH = "/version";
   public static final String OSV_NVD_PURLS_PATH = "/purls";
 
+  public static final String OSV_NVD_HEALTH_PATH = "/q/health";
+
   public static final String TRUSTED_CONTENT_PATH = "/recommend";
   public static final String DEFAULT_ACCEPT_MEDIA_TYPE = MediaType.APPLICATION_JSON;
   public static final boolean DEFAULT_VERBOSE_MODE = false;
diff --git a/src/main/java/com/redhat/exhort/integration/backend/ExhortIntegration.java b/src/main/java/com/redhat/exhort/integration/backend/ExhortIntegration.java
@@ -52,6 +52,7 @@
 import com.redhat.exhort.integration.backend.sbom.SbomParser;
 import com.redhat.exhort.integration.backend.sbom.SbomParserFactory;
 import com.redhat.exhort.integration.providers.ProviderAggregationStrategy;
+import com.redhat.exhort.integration.providers.ProvidersBodyPlusResponseCodeAggregationStrategy;
 import com.redhat.exhort.integration.providers.VulnerabilityProvider;
 import com.redhat.exhort.integration.trustedcontent.TcResponseAggregation;
 import com.redhat.exhort.model.DependencyTree;
@@ -257,7 +258,24 @@ public void configure() {
       .setBody().simple("${exception.message}")
       .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(Status.INTERNAL_SERVER_ERROR.getStatusCode()))
       .setHeader(Exchange.CONTENT_TYPE, constant(MediaType.TEXT_PLAIN));
+
+    from(direct("exhortHealthCheck"))
+      .routeId("exhortHealthCheck")
+      .recipientList(header(Constants.HEALTH_CHECKS_LIST_HEADER_NAME))
+      .aggregationStrategy(new ProvidersBodyPlusResponseCodeAggregationStrategy());
+
+    from(direct("healthCheckProviderDisabled"))
+      .routeId("healthCheckProviderDisabled")
+      .setProperty(Constants.EXCLUDE_FROM_READINESS_CHECK, constant(true))
+      .setBody(constant(String.format("Provider %s is disabled",exchangeProperty(Constants.PROVIDER_NAME))))
+
+      .process(exchange -> {
+        String providerName = exchange.getProperty(Constants.PROVIDER_NAME, String.class);
+        exchange.getMessage().setHeader(Exchange.HTTP_RESPONSE_TEXT,String.format("Provider %s is disabled", providerName)); })
+      .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(Response.Status.SERVICE_UNAVAILABLE));
+
     //fmt:on
+
   }
 
   private void processAnalysisRequest(Exchange exchange) {
diff --git a/src/main/java/com/redhat/exhort/integration/providers/ProviderHealthCheck.java b/src/main/java/com/redhat/exhort/integration/providers/ProviderHealthCheck.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2024 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.redhat.exhort.integration.providers;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import org.apache.camel.builder.ExchangeBuilder;
+import org.apache.camel.health.HealthCheckResultBuilder;
+import org.apache.camel.impl.health.AbstractHealthCheck;
+
+import com.redhat.exhort.api.v4.ProviderStatus;
+import com.redhat.exhort.integration.Constants;
+
+public class ProviderHealthCheck extends AbstractHealthCheck {
+
+  private static final List<String> ALL_PROVIDERS_HEALTH_CHECKS =
+      List.of("direct:snykHealthCheck", "direct:osvNvdHealthCheck", "direct:ossIndexHealthCheck");
+
+  public ProviderHealthCheck() {
+    super("External Providers Readiness Check");
+  }
+
+  @Override
+  protected void doCall(HealthCheckResultBuilder builder, Map<String, Object> options) {
+    var response =
+        getCamelContext()
+            .createProducerTemplate()
+            .send(
+                "direct:exhortHealthCheck",
+                ExchangeBuilder.anExchange(getCamelContext())
+                    .withHeader(
+                        Constants.HEALTH_CHECKS_LIST_HEADER_NAME, this.ALL_PROVIDERS_HEALTH_CHECKS)
+                    .build());
+
+    List<ProviderStatus> httpResponseBodiesAndStatuses =
+        (List<ProviderStatus>) response.getMessage().getBody();
+    Map<String, Object> providers =
+        httpResponseBodiesAndStatuses.stream()
+            .collect(
+                Collectors.toMap(
+                    provider -> provider.getName(),
+                    provider -> formatProviderStatus(provider),
+                    (a, b) -> a));
+    builder.details(providers);
+
+    if (httpResponseBodiesAndStatuses.stream()
+        .filter(providerStatus -> Objects.nonNull(providerStatus.getCode()))
+        .anyMatch(providerDetails -> providerDetails.getCode() < 400 && providerDetails.getOk())) {
+      builder.up();
+
+    } else {
+      builder.down();
+    }
+  }
+
+  private static String formatProviderStatus(ProviderStatus provider) {
+    if (Objects.nonNull(provider.getCode())) {
+      return String.format(
+          "providerName=%s, isEnabled=%s, statusCode=%s, message=%s",
+          provider.getName(), provider.getOk(), provider.getCode(), provider.getMessage());
+    } else {
+      return String.format(
+          "providerName=%s, isEnabled=%s, message=%s",
+          provider.getName(), provider.getOk(), provider.getMessage());
+    }
+  }
+
+  @Override
+  public boolean isLiveness() {
+    return false;
+  }
+}
diff --git a/src/main/java/com/redhat/exhort/integration/providers/ProvidersBodyPlusResponseCodeAggregationStrategy.java b/src/main/java/com/redhat/exhort/integration/providers/ProvidersBodyPlusResponseCodeAggregationStrategy.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2024 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.redhat.exhort.integration.providers;
+
+import java.util.Objects;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Message;
+import org.apache.camel.processor.aggregate.AbstractListAggregationStrategy;
+
+import com.redhat.exhort.api.v4.ProviderStatus;
+import com.redhat.exhort.integration.Constants;
+
+import jakarta.ws.rs.core.Response;
+
+public class ProvidersBodyPlusResponseCodeAggregationStrategy
+    extends AbstractListAggregationStrategy<ProviderStatus> {
+  @Override
+  public ProviderStatus getValue(Exchange exchange) {
+    ProviderStatus providerValues = new ProviderStatus();
+    providerValues.setMessage(getHttpResponseBodyFromMessage(exchange.getMessage()));
+    Integer statusCode = Integer.valueOf(getHttpResponseStatusFromMessage(exchange.getMessage()));
+    if (!serviceExcludedFromReadinessCheck(exchange)) {
+      providerValues.setCode(statusCode);
+    }
+    providerValues.setOk(!serviceExcludedFromReadinessCheck(exchange));
+    String providerName = exchange.getProperty(Constants.PROVIDER_NAME, String.class);
+    providerValues.setName(providerName);
+
+    return providerValues;
+  }
+
+  private static Boolean serviceExcludedFromReadinessCheck(Exchange exchange) {
+    return Objects.requireNonNullElse(
+        exchange.getProperty(Constants.EXCLUDE_FROM_READINESS_CHECK, Boolean.class), false);
+  }
+
+  private static String getHttpResponseStatusFromMessage(Message message) {
+    if (message.getHeader(Exchange.HTTP_RESPONSE_CODE) instanceof Integer) {
+      return message.getHeader(Exchange.HTTP_RESPONSE_CODE).toString();
+    } else {
+      return String.valueOf(
+          message.getHeader(Exchange.HTTP_RESPONSE_CODE, Response.Status.class).getStatusCode());
+    }
+  }
+
+  private static String getHttpResponseBodyFromMessage(Message message) {
+    if (Objects.nonNull(message.getHeader(Exchange.HTTP_RESPONSE_TEXT, String.class))) {
+      return message.getHeader(Exchange.HTTP_RESPONSE_TEXT, String.class);
+    } else {
+      if (Objects.nonNull(message.getBody(String.class))
+          && message.getBody(String.class) instanceof String) {
+        return message.getBody(String.class);
+      } else {
+        return Objects.requireNonNull(
+            message.getHeader(Exchange.HTTP_RESPONSE_CODE, String.class),
+            message.getBody(String.class));
+      }
+    }
+  }
+}
diff --git a/src/main/java/com/redhat/exhort/integration/providers/ossindex/OssIndexIntegration.java b/src/main/java/com/redhat/exhort/integration/providers/ossindex/OssIndexIntegration.java
@@ -35,6 +35,7 @@
 import jakarta.inject.Inject;
 import jakarta.ws.rs.HttpMethod;
 import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 
 @ApplicationScoped
 public class OssIndexIntegration extends EndpointRouteBuilder {
@@ -82,6 +83,31 @@ public void configure() {
         .onFallback()
           .process(responseHandler::processResponseError);
 
+    from(direct("ossIndexHealthCheck"))
+      .routeId("ossIndexHealthCheck")
+      .setProperty(Constants.PROVIDER_NAME, constant(Constants.OSS_INDEX_PROVIDER))
+      .choice()
+        .when(method(vulnerabilityProvider, "getEnabled").contains(Constants.OSS_INDEX_PROVIDER))
+          .to(direct("ossCheckVersionEndpoint"))
+        .otherwise()
+          .to(direct("healthCheckProviderDisabled"));
+
+    from(direct("ossCheckVersionEndpoint"))
+      .routeId("ossCheckVersionEndpoint")
+      .circuitBreaker()
+        .faultToleranceConfiguration()
+           .timeoutEnabled(true)
+           .timeoutDuration(timeout)
+        .end()
+        .process(this::processVersionRequest)
+        .to(vertxHttp("{{api.ossindex.host}}"))
+        .setBody(constant("Service is up and running"))
+        .setHeader(Exchange.HTTP_RESPONSE_TEXT,constant("Service is up and running"))
+        .onFallback()
+           .setBody(constant(Constants.OSS_INDEX_PROVIDER + "Service is down"))
+           .setHeader(Exchange.HTTP_RESPONSE_CODE,constant(Response.Status.SERVICE_UNAVAILABLE))
+      .end();
+
     from(direct("ossValidateCredentials"))
       .routeId("ossValidateCredentials")
       .circuitBreaker()
@@ -120,4 +146,14 @@ private void processComponentRequest(Exchange exchange) {
     exchange.setProperty(
         Constants.AUTH_PROVIDER_REQ_PROPERTY_PREFIX + Constants.OSS_INDEX_PROVIDER, Boolean.TRUE);
   }
+
+  private void processVersionRequest(Exchange exchange) {
+    var message = exchange.getMessage();
+    message.removeHeader(Exchange.HTTP_PATH);
+    message.removeHeader(Exchange.HTTP_QUERY);
+    message.removeHeader(Exchange.HTTP_URI);
+    message.removeHeader(Constants.ACCEPT_ENCODING_HEADER);
+    message.setHeader(Exchange.HTTP_METHOD, HttpMethod.GET);
+    message.setHeader(Exchange.HTTP_PATH, Constants.OSS_INDEX_VERSION_PATH);
+  }
 }
diff --git a/src/main/java/com/redhat/exhort/integration/providers/osvnvd/OsvNvdIntegration.java b/src/main/java/com/redhat/exhort/integration/providers/osvnvd/OsvNvdIntegration.java
@@ -23,18 +23,21 @@
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 
 import com.redhat.exhort.integration.Constants;
+import com.redhat.exhort.integration.providers.VulnerabilityProvider;
 
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import jakarta.ws.rs.HttpMethod;
 import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 
 @ApplicationScoped
 public class OsvNvdIntegration extends EndpointRouteBuilder {
 
   @ConfigProperty(name = "api.osvnvd.timeout", defaultValue = "10s")
   String timeout;
 
+  @Inject VulnerabilityProvider vulnerabilityProvider;
   @Inject OsvNvdResponseHandler responseHandler;
 
   @Override
@@ -59,18 +62,51 @@ public void configure() throws Exception {
       .process(this::processRequest)
       .to(vertxHttp("{{api.osvnvd.host}}"))
       .transform().method(responseHandler, "responseToIssues");
+
+    from(direct("osvNvdHealthCheck"))
+      .routeId("osvNvdHealthCheck")
+      .setProperty(Constants.PROVIDER_NAME, constant(Constants.OSV_NVD_PROVIDER))
+      .choice()
+         .when(method(vulnerabilityProvider, "getEnabled").contains(Constants.OSV_NVD_PROVIDER))
+            .to(direct("osvNvdHealthCheckEndpoint"))
+         .otherwise()
+            .to(direct("healthCheckProviderDisabled"));
+
+    from(direct("osvNvdHealthCheckEndpoint"))
+      .routeId("osvNvdHealthCheckEndpoint")
+      .process(this::processHealthRequest)
+      .circuitBreaker()
+         .faultToleranceConfiguration()
+            .timeoutEnabled(true)
+            .timeoutDuration(timeout)
+         .end()
+         .to(vertxHttp("{{api.osvnvd.management.host}}"))
+         .setHeader(Exchange.HTTP_RESPONSE_TEXT,constant("Service is up and running"))
+         .setBody(constant("Service is up and running"))
+      .onFallback()
+         .setBody(constant(Constants.OSV_NVD_PROVIDER + "Service is down"))
+         .setHeader(Exchange.HTTP_RESPONSE_CODE,constant(Response.Status.SERVICE_UNAVAILABLE))
+      .end();
     // fmt:on
   }
 
   private void processRequest(Exchange exchange) {
     var message = exchange.getMessage();
     message.removeHeader(Exchange.HTTP_QUERY);
     message.removeHeader(Exchange.HTTP_URI);
-    message.removeHeader("Accept-Encoding");
+    message.removeHeader(Constants.ACCEPT_ENCODING_HEADER);
     message.setHeader(Exchange.CONTENT_TYPE, MediaType.APPLICATION_JSON);
     message.setHeader(Exchange.HTTP_PATH, Constants.OSV_NVD_PURLS_PATH);
     message.setHeader(Exchange.HTTP_METHOD, HttpMethod.POST);
-    exchange.setProperty(
-        Constants.AUTH_PROVIDER_REQ_PROPERTY_PREFIX + Constants.OSV_NVD_PROVIDER, Boolean.FALSE);
+  }
+
+  private void processHealthRequest(Exchange exchange) {
+    var message = exchange.getMessage();
+    message.removeHeader(Exchange.HTTP_QUERY);
+    message.removeHeader(Exchange.HTTP_URI);
+    message.removeHeader(Constants.ACCEPT_ENCODING_HEADER);
+    message.removeHeader(Exchange.CONTENT_TYPE);
+    message.setHeader(Exchange.HTTP_PATH, Constants.OSV_NVD_HEALTH_PATH);
+    message.setHeader(Exchange.HTTP_METHOD, HttpMethod.GET);
   }
 }
diff --git a/src/main/java/com/redhat/exhort/integration/providers/snyk/SnykIntegration.java b/src/main/java/com/redhat/exhort/integration/providers/snyk/SnykIntegration.java
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
