WIP: Add registry-watcher image and separate docker-compose services

.docker.env.sample

@@ -0,0 +1,12 @@
+# environment overrides for the app instances running inside docker compose
+DOCSRS_LOG=docs_rs=debug,rustwide=info
+DOCSRS_INCLUDE_DEFAULT_TARGETS=false
+DOCSRS_DOCKER_IMAGE=ghcr.io/rust-lang/crates-build-env/linux-micro
+# To build with a PR that hasn't landed in a rust dist toolchain yet,
+# you can set this to the git sha of a try build:
+# https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds
+DOCSRS_TOOLCHAIN=nightly
+
+# for the registry watcher, automatically queued reqbuidls.
+DOCSRS_MAX_QUEUED_REBUILDS: 10
+

.dockerignore

@@ -1,4 +1,19 @@
+/target/
 /.rustwide
 /.rustwide-docker
+/Justfile
+/justfiles/
+/LICENSE
+/README.md
+/docker-compose.yml
+/dockerfiles/
+/docs/
 /ignored
-**/target
+/mcps
+/triagebot.toml
+/clippy.toml
+/.env
+/.env.*
+/.envrc
+/.docker.env
+archive_cache

.editorconfig

@@ -8,5 +8,9 @@ insert_final_newline = true
 indent_style = space
 indent_size = 4
 
+[Justfile]
+indent_size = 2
+
 [*.js]
 max_line_length = 100
+

.env.sample

@@ -1,18 +1,25 @@
-export DOCSRS_PREFIX=ignored/cratesfyi-prefix
-export DOCSRS_DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
+# environment overrides for local development outside docker.
+DOCSRS_PREFIX=ignored/cratesfyi-prefix
+DOCSRS_DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
 
 # for local development with sqlx
-export DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
+DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
 
-export DOCSRS_LOG=docs_rs=debug,rustwide=info
+DOCSRS_LOG=docs_rs=debug,rustwide=info
+AWS_ACCESS_KEY_ID=cratesfyi
+AWS_SECRET_ACCESS_KEY=secret_key
+S3_ENDPOINT=http://localhost:9000
+DOCSRS_S3_STATIC_ROOT_PATH=http://localhost:9000/rust-docs-rs
+
+
+# build specific
+DOCSRS_INCLUDE_DEFAULT_TARGETS=false
+DOCSRS_DOCKER_IMAGE=ghcr.io/rust-lang/crates-build-env/linux-micro
+SENTRY_ENVIRONMENT=dev
 # To build with a PR that hasn't landed in a rust dist toolchain yet,
 # you can set this to the git sha of a try build:
 # https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds
-export DOCSRS_TOOLCHAIN=nightly
-export AWS_ACCESS_KEY_ID=cratesfyi
-export AWS_SECRET_ACCESS_KEY=secret_key
-export S3_ENDPOINT=http://localhost:9000
-export DOCSRS_S3_STATIC_ROOT_PATH=http://localhost:9000/rust-docs-rs
-export DOCSRS_INCLUDE_DEFAULT_TARGETS=false
-export DOCSRS_DOCKER_IMAGE=ghcr.io/rust-lang/crates-build-env/linux-micro
-export SENTRY_ENVIRONMENT=dev
+DOCSRS_TOOLCHAIN=nightly
+
+# NOTE: when running services in docker-compose, you can override the settings in 
+# `.docker.env`, you'll fine an example in `.docker.env.sample`.

.github/workflows/ci.yml

@@ -9,152 +9,87 @@ on:
     - cron: "0 0 * * *"
 
 env:
-  RUST_BACKTRACE: 1
   RUST_CACHE_KEY: rust-cache-20241114
-  DOCSRS_PREFIX: ignored/cratesfyi-prefix
-  DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@localhost:15432
-  DOCSRS_LOG: docs_rs=debug,rustwide=info
-  AWS_ACCESS_KEY_ID: cratesfyi
-  AWS_SECRET_ACCESS_KEY: secret_key
-  S3_ENDPOINT: http://localhost:9000
-  DOCSRS_INCLUDE_DEFAULT_TARGETS: false
-  DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro
-  SENTRY_ENVIRONMENT: dev
 
 jobs:
   sqlx:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
-      # We check that all github actions workflows have valid syntax
-      - name: Validate YAML file
-        uses: raven-actions/actionlint@v2
-        with:
-          files: .github/workflow/*
-          flags: "-ignore SC2086" # ignore some shellcheck errors
-
       - name: install `just`
         run: sudo snap install --edge --classic just
 
+      - uses: cargo-bins/cargo-binstall@main
+
       - name: restore build & cargo cache
         uses: Swatinem/rust-cache@v2
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
-      - name: Launch postgres
-        run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: install SQLX CLI
-        run: cargo install sqlx-cli --no-default-features --features postgres
-
-      - name: run database migrations
-        run: cargo sqlx migrate run --database-url $DOCSRS_DATABASE_URL
+        run: cargo binstall sqlx-cli
 
-      - name: run sqlx prepare --check
-        run: just sqlx-prepare
-
-      - name: test reverse migrations
+      - name: run sqlx migration up & down
         run: |
-          # --target 0 means "revert everything"
-          cargo sqlx migrate revert \
-            --database-url $DOCSRS_DATABASE_URL \
-            --target-version 0
+          just sqlx-migrate-run \
+            sqlx-check \
+            sqlx-migrate-revert
 
-      - name: Clean up the database
-        run: docker compose down --volumes
+      - name: shut down test environment
+        if: ${{ always() }}
+        run: just compose-down-and-wipe
 
   test:
-    env:
-      SQLX_OFFLINE: 1
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
+      - name: install `just`
+        run: sudo snap install --edge --classic just
+
       - name: restore build & cargo cache
         uses: Swatinem/rust-cache@v2
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
-      - name: Build
-        run: cargo build --workspace --locked
-
-      - name: Launch postgres and min.io
-        run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
-
-      - name: run workspace tests
-        run: |
-          cargo test --workspace --locked --no-fail-fast
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: run slow tests
-        env:
-          DOCSRS_INCLUDE_DEFAULT_TARGETS: true
-        run: |
-          cargo test --locked -- --ignored --test-threads=1
+      - name: run tests
+        run: just run-tests run-builder-tests
 
-      - name: Clean up the database
-        run: docker compose down --volumes
+      - name: shut down test environment
+        if: ${{ always() }}
+        run: just compose-down-and-wipe
 
   GUI_test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
-      - name: restore build & cargo cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: ${{ env.RUST_CACHE_KEY }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: Launch postgres and min.io
-        run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+      - name: install `just`
+        run: sudo snap install --edge --classic just
 
       - name: Run GUI tests
-        run: ./dockerfiles/run-gui-tests.sh
+        run: just run-gui-tests
 
-      - name: Clean up the database
-        run: docker compose down --volumes
+      - name: shut down test environment
+        if: ${{ always() }}
+        run: just compose-down-and-wipe
 
-  fmt:
-    name: Rustfmt
+  lint-rs:
+    name: rust linters
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v5
-      - name: update rust toolchain
-        run: rustup component add rustfmt
-
-      - run: cargo fmt -- --check
-
-  clippy:
-    name: Clippy
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: update rust toolchain
-        run: rustup component add clippy
 
       - name: install `just`
         run: sudo snap install --edge --classic just
@@ -164,13 +99,33 @@ jobs:
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - run: just lint
 
-  eslint:
+  lint-js:
+    name: js linters
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
+
       - uses: denoland/setup-deno@v2
+
       - name: install `just`
         run: sudo snap install --edge --classic just
+
       - run: just lint-js
+
+  lint-actions:
+    name: gh actions linters
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      # We check that all github actions workflows have valid syntax
+      - name: Validate YAML file
+        uses: raven-actions/actionlint@v2
+        with:
+          files: .github/workflow/*
+          flags: "-ignore SC2086"  # ignore some shellcheck errors

.github/workflows/docker.yml

@@ -1,13 +1,47 @@
 name: Docker
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  schedule:
+    - cron: "0 0 * * *"
 
 jobs:
   docker:
+    strategy:
+      matrix:
+        target: [
+          "web-server",
+          "build-server",
+          "registry-watcher",
+          "cli"
+        ]
     name: Test docker image builds
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
-      - name: Build the Docker image
-        run: docker build -t docs-rs -f dockerfiles/Dockerfile .
+      - name: setup docker buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: build docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: "./dockerfiles/Dockerfile"
+          platforms: linux/amd64
+          target: ${{ matrix.target }}
+          build-args: |
+            GIT_SHA=${{ github.sha }}
+            PROFILE=release
+            PROFILE_DIR=release
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
+
+          # TODO: later we would set `push: true` and also provide nice tags
+          # for the images.
+          # Unclear is how the deploy would work then.