Mlflow Release Pipeline #31
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MLflow CI | |
| on: | |
| pull_request: | |
| paths: | |
| - 'applications/mlflow/charts/**' | |
| - 'applications/mlflow/kots/**' | |
| - 'applications/mlflow/tests/**' | |
| - '.github/workflows/mlflow-ci.yml' | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'applications/mlflow/charts/**' | |
| - 'applications/mlflow/kots/**' | |
| - 'applications/mlflow/tests/**' | |
| - '.github/workflows/mlflow-ci.yml' | |
| env: | |
| APP_SLUG: diamon-mlflow | |
| jobs: | |
| lint-and-template: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Helm | |
| uses: azure/[email protected] | |
| with: | |
| version: v3.13.3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.12 | |
| - name: Add Helm repositories | |
| run: | | |
| cd applications/mlflow | |
| make add-helm-repositories | |
| - name: Lint charts | |
| run: | | |
| cd applications/mlflow | |
| make lint | |
| - name: Template charts with SDK disabled | |
| run: | | |
| cd applications/mlflow | |
| make template | |
| - name: Upload templates (if templating failed) | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: failed-templates | |
| path: applications/mlflow/rendered-templates | |
| if-no-files-found: warn | |
| create-release: | |
| runs-on: ubuntu-22.04 | |
| needs: [lint-and-template] | |
| outputs: | |
| customer-id: ${{ steps.create-customer.outputs.customer-id }} | |
| channel-slug: ${{ steps.create-release.outputs.channel-slug }} | |
| chart-version: ${{ steps.chart-version.outputs.chart_version }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Helm | |
| uses: azure/[email protected] | |
| with: | |
| version: v3.13.3 | |
| # Add required Helm repositories | |
| - name: Add Helm repositories | |
| run: | | |
| cd applications/mlflow | |
| make add-helm-repositories | |
| - name: Package infra chart | |
| run: | | |
| helm package applications/mlflow/charts/infra -d applications/mlflow/kots/ -u | |
| if [ ! -f applications/mlflow/kots/infra-*.tgz ]; then | |
| echo "Error: Infra chart packaging failed" | |
| exit 1 | |
| fi | |
| - name: Package mlflow chart | |
| run: | | |
| helm package applications/mlflow/charts/mlflow -d applications/mlflow/kots/ -u | |
| if [ ! -f applications/mlflow/kots/mlflow-*.tgz ]; then | |
| echo "Error: MLflow chart packaging failed" | |
| exit 1 | |
| fi | |
| # The following steps implement our versioning strategy: | |
| # 1. We extract the chart version from mlflow-chart.yaml | |
| # 2. We use this version for the Replicated release | |
| # This ensures that the Replicated release version always matches the MLflow chart version | |
| - name: Extract MLflow chart version | |
| id: chart-version | |
| run: | | |
| CHART_VERSION=$(grep 'chartVersion:' applications/mlflow/kots/mlflow-chart.yaml | awk '{print $2}') | |
| echo "chart_version=$CHART_VERSION" >> $GITHUB_OUTPUT | |
| echo "Using MLflow chart version: $CHART_VERSION" | |
| - name: Create release | |
| id: create-release | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| with: | |
| app-slug: ${{ env.APP_SLUG }} | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| yaml-dir: applications/mlflow/kots/ | |
| promote-channel: ci-automation-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} | |
| version: ${{ steps.chart-version.outputs.chart_version }} | |
| - name: Create customer | |
| id: create-customer | |
| uses: replicatedhq/replicated-actions/create-customer@main | |
| with: | |
| app-slug: ${{ env.APP_SLUG }} | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| customer-name: automated-${{ github.run_id }} | |
| customer-email: [email protected] | |
| license-type: dev | |
| channel-slug: ${{ steps.create-release.outputs.channel-slug }} | |
| is-kots-install-enabled: "true" | |
| helm-install-test: | |
| runs-on: ubuntu-22.04 | |
| needs: [create-release] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cluster: | |
| - distribution: kind | |
| version: 1.32 | |
| #- distribution: kind | |
| # version: 1.31 | |
| #- distribution: kind | |
| # version: 1.30 | |
| config: | |
| - name: nodeport-ingress-disabled | |
| values_file: tests/helm/nodeport-ingress-disabled.yaml | |
| port: 30080 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Helm | |
| uses: azure/[email protected] | |
| with: | |
| version: v3.13.3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.12 | |
| # Install jq via apt-get | |
| - name: Install jq | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y jq | |
| # Get license ID from customer inspect | |
| - name: Get License ID | |
| id: get-license | |
| run: | | |
| # Run vendor-cli to inspect the customer and get the installation ID | |
| CUSTOMER_JSON=$(docker run --rm \ | |
| -e REPLICATED_API_TOKEN=${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} \ | |
| -e REPLICATED_APP=${{ env.APP_SLUG }} \ | |
| replicated/vendor-cli:latest \ | |
| customer inspect --customer "automated-${{ github.run_id }}" --output json) | |
| # Use jq to properly extract the installationId | |
| INSTALLATION_ID=$(echo "$CUSTOMER_JSON" | jq -r '.installationId') | |
| if [ -z "$INSTALLATION_ID" ] || [ "$INSTALLATION_ID" = "null" ]; then | |
| echo "Failed to extract installationId from customer JSON" | |
| echo "JSON structure:" | |
| echo "$CUSTOMER_JSON" | jq 'del(.installationId)' # Print JSON without the license ID | |
| exit 1 | |
| fi | |
| # Don't print the actual license ID, just indicate success | |
| echo "Successfully extracted installationId" | |
| echo "license_id=$INSTALLATION_ID" >> $GITHUB_OUTPUT | |
| - name: Create Cluster | |
| id: create-cluster | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| with: | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| kubernetes-distribution: ${{ matrix.cluster.distribution }} | |
| kubernetes-version: ${{ matrix.cluster.version }} | |
| cluster-name: mlflow-ci-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}-${{ matrix.config.name }} | |
| ttl: 1h | |
| export-kubeconfig: true | |
| - name: Expose Application Port | |
| id: expose-port | |
| uses: replicatedhq/replicated-actions/expose-port@main | |
| with: | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} | |
| port: '${{ matrix.config.port }}' | |
| protocols: 'http,https' | |
| wildcard: 'false' | |
| timeout-minutes: '5' | |
| - name: Add Helm repositories | |
| run: | | |
| cd applications/mlflow | |
| make add-helm-repositories | |
| # Authenticate with the Replicated registry | |
| - name: Login to Replicated registry | |
| run: | | |
| echo "Authenticating with Replicated registry using license ID from previous step" | |
| cd applications/mlflow | |
| # Print env var name to check it's set in the shell (without revealing value) | |
| if [ -n "$REPLICATED_LICENSE_ID" ]; then | |
| echo "REPLICATED_LICENSE_ID is set in the shell" | |
| else | |
| echo "ERROR: REPLICATED_LICENSE_ID is not set in the shell" | |
| exit 1 | |
| fi | |
| # Pass the env var directly to make | |
| REPLICATED_LICENSE_ID="$REPLICATED_LICENSE_ID" make registry-login | |
| env: | |
| REPLICATED_LICENSE_ID: ${{ steps.get-license.outputs.license_id }} | |
| - name: Run Helm installation test with custom values | |
| run: | | |
| cd applications/mlflow | |
| # Save kubeconfig to a file | |
| KUBECONFIG_FILE="/tmp/kubeconfig-${{ github.run_id }}" | |
| echo "$KUBECONFIG" > "$KUBECONFIG_FILE" | |
| echo "Saved kubeconfig to $KUBECONFIG_FILE" | |
| # Set up environment for the make target | |
| export KUBECONFIG="$KUBECONFIG_FILE" | |
| export REPLICATED_APP="${REPLICATED_APP}" | |
| export REPLICATED_CHANNEL="${REPLICATED_CHANNEL}" | |
| export REPLICATED_LICENSE_ID="${REPLICATED_LICENSE_ID}" | |
| # Use test-specific values file | |
| export MLFLOW_VALUES="${{ matrix.config.values_file }}" | |
| echo "Running test '${{ matrix.config.name }}' with MLflow values file: $MLFLOW_VALUES" | |
| # Run direct Helm installation | |
| make test-replicated-helm-with-values | |
| env: | |
| KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }} | |
| REPLICATED_APP: ${{ env.APP_SLUG }} | |
| REPLICATED_CHANNEL: ${{ needs.create-release.outputs.channel-slug }} | |
| REPLICATED_LICENSE_ID: ${{ steps.get-license.outputs.license_id }} | |
| # Application testing with our consolidated test file | |
| - name: Run Application Tests | |
| run: | | |
| cd applications/mlflow | |
| echo "Installing Python dependencies for tests..." | |
| pip install mlflow pandas scikit-learn | |
| echo "Running MLflow application tests against ${{ steps.expose-port.outputs.hostname }}" | |
| python tests/mlflow_test.py ${{ steps.expose-port.outputs.hostname }} --protocol https | |
| if: false # Disabled for now until we're ready to implement application tests | |
| - name: Install troubleshoot | |
| run: curl -L https://github.com/replicatedhq/troubleshoot/releases/latest/download/support-bundle_linux_amd64.tar.gz | tar xzvf - | |
| if: failure() | |
| - name: Collect bundle | |
| run: | | |
| # Save kubeconfig to a file | |
| KUBECONFIG_FILE="/tmp/kubeconfig-helm-bundle-${{ github.run_id }}" | |
| echo "$KUBECONFIG" > "$KUBECONFIG_FILE" | |
| echo "Saved kubeconfig to $KUBECONFIG_FILE" | |
| ./support-bundle --kubeconfig="$KUBECONFIG_FILE" --interactive=false -o ci-bundle-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}-${{ matrix.config.name }} https://raw.githubusercontent.com/replicatedhq/troubleshoot-specs/main/in-cluster/default.yaml | |
| if: failure() | |
| env: | |
| KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }} | |
| - name: Upload support bundle artifact | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: mlflow-bundle-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}-${{ matrix.config.name }} | |
| path: 'ci-bundle-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}-${{ matrix.config.name }}.tar.gz' | |
| - name: Remove Cluster | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} | |
| with: | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} | |
| kots-install-test: | |
| runs-on: ubuntu-22.04 | |
| needs: [create-release] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cluster: | |
| - distribution: kind | |
| version: 1.32 | |
| #- distribution: kind | |
| # version: 1.31 | |
| #- distribution: kind | |
| # version: 1.30 | |
| #- distribution: aks | |
| # version: 1.31 | |
| #- distribution: aks | |
| # version: 1.30 | |
| #- distribution: gke | |
| # version: 1.32 | |
| #- distribution: gke | |
| # version: 1.31 | |
| #- distribution: gke | |
| # version: 1.30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Create Cluster | |
| id: create-cluster | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| with: | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| kubernetes-distribution: ${{ matrix.cluster.distribution }} | |
| kubernetes-version: ${{ matrix.cluster.version }} | |
| cluster-name: mlflow-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} | |
| ttl: 1h | |
| export-kubeconfig: true | |
| # Download license using Replicated vendor-cli Docker container | |
| - name: Download license | |
| id: download-license | |
| run: | | |
| # Create a temporary file to store the license | |
| mkdir -p /tmp/replicated | |
| # Run the vendor-cli command and capture its output | |
| docker run --rm \ | |
| -e REPLICATED_API_TOKEN=${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} \ | |
| -e REPLICATED_APP=${{ env.APP_SLUG }} \ | |
| replicated/vendor-cli:latest \ | |
| customer download-license --customer ${{ needs.create-release.outputs.customer-id }} > /tmp/replicated/license.yaml | |
| # Read the license and set it as an output | |
| LICENSE_CONTENT=$(cat /tmp/replicated/license.yaml) | |
| # Use EOF delimiter for multi-line output | |
| echo "license<<EOF" >> $GITHUB_OUTPUT | |
| echo "$LICENSE_CONTENT" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| # Install using KOTS | |
| - name: KOTS Install | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| with: | |
| kubeconfig: ${{ steps.create-cluster.outputs.cluster-kubeconfig }} | |
| kots-version: latest | |
| app-slug: ${{ env.APP_SLUG }}/${{ needs.create-release.outputs.channel-slug }} | |
| app-version-label: ${{ needs.create-release.outputs.chart-version }} | |
| license-file: ${{ steps.download-license.outputs.license }} | |
| namespace: default | |
| wait-duration: 10m | |
| shared-password: 'replicatedmlflow' | |
| - name: Install troubleshoot | |
| run: curl -L https://github.com/replicatedhq/troubleshoot/releases/latest/download/support-bundle_linux_amd64.tar.gz | tar xzvf - | |
| if: failure() | |
| - name: Collect bundle | |
| run: | | |
| # Save kubeconfig to a file | |
| KUBECONFIG_FILE="/tmp/kubeconfig-kots-bundle-${{ github.run_id }}" | |
| echo "$KUBECONFIG" > "$KUBECONFIG_FILE" | |
| echo "Saved kubeconfig to $KUBECONFIG_FILE" | |
| ./support-bundle --kubeconfig="$KUBECONFIG_FILE" --interactive=false -o kots-ci-bundle-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} https://raw.githubusercontent.com/replicatedhq/troubleshoot-specs/main/in-cluster/default.yaml | |
| if: failure() | |
| env: | |
| KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }} | |
| - name: Upload support bundle artifact | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: mlflow-kots-bundle-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} | |
| path: 'kots-ci-bundle-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}.tar.gz' | |
| - name: Remove Cluster | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} | |
| with: | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} | |
| cleanup-test-release: | |
| runs-on: ubuntu-22.04 | |
| needs: [create-release, kots-install-test, helm-install-test] | |
| if: always() | |
| steps: | |
| - name: Archive Customer | |
| if: ${{ always() && needs.create-release.outputs.customer-id != '' }} | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| with: | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| customer-id: ${{ needs.create-release.outputs.customer-id }} | |
| - name: Archive Channel | |
| if: ${{ always() && needs.create-release.outputs.channel-slug != '' }} | |
| uses: replicatedhq/replicated-actions/[email protected] | |
| with: | |
| app-slug: ${{ env.APP_SLUG }} | |
| api-token: ${{ secrets.REPLICATED_PLATFORM_EXAMPLES_TOKEN }} | |
| channel-slug: ${{ needs.create-release.outputs.channel-slug }} |