diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6df75834..89d10f22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,158 +6,11 @@ on: workflow_dispatch: jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - components: rustfmt, clippy - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: pre-commit/action@v3.0.1 - with: - extra_args: --all-files - - - uses: taiki-e/install-action@v2 - with: - tool: bindgen-cli@0.72.1 - - name: Check if bindings are up-to-date - working-directory: crates/codspeed/src/instrument_hooks - run: | - ./update-bindings.sh - - if ! git diff --exit-code bindings.rs; then - echo "Error: FFI bindings are out of date!" - exit 1 - fi - - test-codspeed: - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest, target: arm-unknown-linux-gnueabihf } - - { os: ubuntu-latest, target: aarch64-unknown-linux-musl } - - { os: ubuntu-latest, target: i686-unknown-linux-gnu } - - { os: ubuntu-latest, target: i686-unknown-linux-musl } - - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } - - { os: ubuntu-latest, target: x86_64-unknown-linux-musl } - - { os: macos-latest, target: aarch64-apple-darwin } - - { os: macos-latest, target: x86_64-apple-darwin } - - { os: windows-latest, target: i686-pc-windows-msvc } - - { os: windows-latest, target: x86_64-pc-windows-msvc } - - { os: windows-latest, target: aarch64-pc-windows-msvc } - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - uses: moonrepo/setup-rust@v1 - with: - targets: ${{ matrix.job.target }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: taiki-e/install-action@v2 - with: - tool: cross@0.2.5 - - - name: Build codspeed - run: cross build -p codspeed --target ${{ matrix.job.target }} - - msrv-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - bins: cargo-msrv - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Check cospeed MSRV - run: cargo msrv --path crates/codspeed verify -- cargo check --all-features --config codspeed=true - - name: Check bencher_compat MSRV - run: cargo msrv --path crates/bencher_compat verify -- cargo check --all-features --config codspeed=true - - name: Check criterion_compat MSRV - run: cargo msrv --path crates/criterion_compat verify -- cargo check --all-features --config codspeed=true - - name: Check divan_compat MSRV - run: cargo msrv --path crates/divan_compat verify -- cargo check --all-features --config codspeed=true - - tests-without-cargo-codspeed: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - bins: cargo-nextest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: cargo nextest run --workspace --exclude cargo-codspeed - - test-cargo-codspeed: - runs-on: ubuntu-latest - strategy: - matrix: - partition: [1, 2, 3, 4, 5] - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - bins: cargo-nextest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/5 - - compat-integration-test-instrumentation: - runs-on: ubuntu-latest - strategy: - matrix: - build-args: - - "-p codspeed" - - "-p codspeed-bencher-compat" - - "--features async_futures -p codspeed-criterion-compat" - - "-p codspeed-divan-compat" - - "-p codspeed-divan-compat-examples" - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - cache-target: release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - run: cargo install --path crates/cargo-codspeed --locked - - - run: cargo codspeed build ${{ matrix.build-args }} - - - name: Run the benchmarks - uses: CodSpeedHQ/action@main - env: - MY_ENV_VAR: "YES" - with: - run: cargo codspeed run - mode: instrumentation - token: ${{ secrets.CODSPEED_TOKEN }} - compat-integration-test-walltime: runs-on: codspeed-macro strategy: matrix: package: - - codspeed-divan-compat - - codspeed-divan-compat-examples - codspeed-criterion-compat steps: - uses: actions/checkout@v4 @@ -184,45 +37,3 @@ jobs: run: cargo codspeed run mode: walltime token: ${{ secrets.CODSPEED_TOKEN }} - - musl-build-check: - strategy: - matrix: - include: - - target: x86_64-unknown-linux-musl - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-musl - runner: codspeedhq-arm64-ubuntu-24.04 - - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - targets: ${{ matrix.target }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install musl tools - run: sudo apt-get update && sudo apt-get install -y musl-tools - - - run: cargo build --locked --release --bin cargo-codspeed --target ${{ matrix.target }} - - check: - runs-on: ubuntu-latest - if: always() - needs: - - lint - - test-codspeed - - tests-without-cargo-codspeed - - test-cargo-codspeed - - msrv-check - - compat-integration-test-instrumentation - - compat-integration-test-walltime - - musl-build-check - steps: - - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJson( needs ) }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 2bd00512..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: Release on tag - -on: - push: - tags: - - "v*" - workflow_dispatch: - -permissions: - contents: write - -jobs: - build-musl-binaries: - strategy: - matrix: - include: - - target: x86_64-unknown-linux-musl - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-musl - runner: codspeedhq-arm64-ubuntu-24.04 - - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - uses: moonrepo/setup-rust@v1 - with: - targets: ${{ matrix.target }} - - - name: Install musl tools - run: sudo apt-get update && sudo apt-get install -y musl-tools - - - run: cargo build --locked --release --bin cargo-codspeed --target ${{ matrix.target }} - - - name: Upload binary as artifact - uses: actions/upload-artifact@v4 - with: - name: cargo-codspeed-${{ matrix.target }} - path: ./target/${{ matrix.target }}/release/cargo-codspeed - if-no-files-found: error - - publish: - needs: build-musl-binaries - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - uses: moonrepo/setup-rust@v0 - with: - cache-target: release - bins: cargo-workspaces - - name: Build - run: cargo build --release - - name: Publish package - if: github.event_name != 'workflow_dispatch' - run: cargo workspaces publish --from-git - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - - name: Create a draft release - id: create_release - if: github.event_name != 'workflow_dispatch' - run: | - NEW_VERSION=$(cargo workspaces ls --json | jq -r '.[] | select(.name == "codspeed") | .version') - gh release create v$NEW_VERSION --title "v$NEW_VERSION" --generate-notes -d - echo "upload_url=$(gh release view v$NEW_VERSION --json uploadUrl | jq -r '.uploadUrl')" >> $GITHUB_OUTPUT - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} - - upload-binaries: - needs: publish - strategy: - fail-fast: false - matrix: - target: - - x86_64-unknown-linux-musl - - aarch64-unknown-linux-musl - - runs-on: ubuntu-latest - steps: - - name: Download binary artifact - uses: actions/download-artifact@v4 - with: - name: cargo-codspeed-${{ matrix.target }} - path: ./target/${{ matrix.target }}/release - - - name: Upload Release Asset - if: github.event_name != 'workflow_dispatch' - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.publish.outputs.upload_url }} - asset_path: ./target/${{ matrix.target }}/release/cargo-codspeed - asset_name: cargo-codspeed-${{ matrix.target }} - asset_content_type: application/octet-stream diff --git a/crates/criterion_compat/criterion_fork/src/plot/plotters_backend/summary.rs b/crates/criterion_compat/criterion_fork/src/plot/plotters_backend/summary.rs index 0ebb851e..e43cd134 100644 --- a/crates/criterion_compat/criterion_fork/src/plot/plotters_backend/summary.rs +++ b/crates/criterion_compat/criterion_fork/src/plot/plotters_backend/summary.rs @@ -28,12 +28,44 @@ pub fn line_comparison( value_type: ValueType, axis_scale: AxisScale, ) { + eprintln!("[DIAG] line_comparison: title = {}", title); let (unit, series_data) = line_comparison_series_data(formatter, all_curves); + eprintln!( + "[DIAG] line_comparison: series_data count = {}", + series_data.len() + ); + for (i, (name, xs, ys)) in series_data.iter().enumerate() { + eprintln!( + "[DIAG] line_comparison: series[{}] name={:?}, xs len={}, ys len={}", + i, + name, + xs.len(), + ys.len() + ); + if ys.iter().any(|y| !y.is_finite()) { + eprintln!( + "[DIAG] line_comparison: WARNING - series[{}] has non-finite y values!", + i + ); + } + } + let x_range = plotters::data::fitting_range(series_data.iter().flat_map(|(_, xs, _)| xs.iter())); let y_range = plotters::data::fitting_range(series_data.iter().flat_map(|(_, _, ys)| ys.iter())); + + eprintln!( + "[DIAG] line_comparison: x_range = {:?}, y_range = {:?}", + x_range, y_range + ); + eprintln!( + "[DIAG] line_comparison: x_range.is_finite = {}, y_range.is_finite = {}", + x_range.start.is_finite() && x_range.end.is_finite(), + y_range.start.is_finite() && y_range.end.is_finite() + ); + let root_area = SVGBackend::new(&path, SIZE) .into_drawing_area() .titled(&format!("{}: Comparison", title), (DEFAULT_FONT, 20)) @@ -118,13 +150,41 @@ fn line_comparison_series_data<'a>( formatter: &dyn ValueFormatter, all_curves: &[&(&'a BenchmarkId, Vec)], ) -> (&'static str, Vec<(Option<&'a String>, Vec, Vec)>) { - let max = all_curves + eprintln!( + "[DIAG] line_comparison_series_data: Processing {} curves", + all_curves.len() + ); + + let means: Vec = all_curves .iter() - .map(|&(_, data)| Sample::new(data).mean()) - .fold(::std::f64::NAN, f64::max); + .enumerate() + .map(|(i, &(id, data))| { + let mean = Sample::new(data).mean(); + if !mean.is_finite() { + eprintln!("[DIAG] line_comparison_series_data: Curve {} (id={}) has non-finite mean: {}, data len: {}", + i, id.as_title(), mean, data.len()); + } + mean + }) + .collect(); + + let max = means.iter().fold(::std::f64::NAN, |acc, &x| acc.max(x)); + + eprintln!( + "[DIAG] line_comparison_series_data: max = {}, is_finite = {}", + max, + max.is_finite() + ); + if !max.is_finite() { + eprintln!("[DIAG] line_comparison_series_data: All means: {:?}", means); + } let mut dummy = [1.0]; let unit = formatter.scale_values(max, &mut dummy); + eprintln!( + "[DIAG] line_comparison_series_data: unit = {}, dummy after scale = {:?}", + unit, dummy + ); let mut series_data = vec![]; @@ -158,14 +218,57 @@ pub fn violin( path: &Path, axis_scale: AxisScale, ) { + eprintln!( + "[DIAG] violin: title = {}, processing {} curves", + title, + all_curves.len() + ); let all_curves_vec = all_curves.iter().rev().cloned().collect::>(); let all_curves: &[&(&BenchmarkId, Vec)] = &all_curves_vec; let mut kdes = all_curves .iter() - .map(|&&(id, ref sample)| { - let (x, mut y) = kde::sweep(Sample::new(sample), KDE_POINTS, None); + .enumerate() + .map(|(i, &&(id, ref sample))| { + eprintln!( + "[DIAG] violin: Curve {} (id={}): sample len = {}", + i, + id.as_title(), + sample.len() + ); + let sample_obj = Sample::new(sample); + eprintln!( + "[DIAG] violin: Curve {} min={}, max={}, mean={}", + i, + sample_obj.min(), + sample_obj.max(), + sample_obj.mean() + ); + + let (x, mut y) = kde::sweep(sample_obj, KDE_POINTS, None); + eprintln!( + "[DIAG] violin: Curve {} KDE produced {} x-points and {} y-points", + i, + x.len(), + y.len() + ); + let y_max = Sample::new(&y).max(); + eprintln!( + "[DIAG] violin: Curve {} y_max = {}, is_finite = {}", + i, + y_max, + y_max.is_finite() + ); + + if !y_max.is_finite() || y_max == 0.0 { + eprintln!( + "[DIAG] violin: WARNING - Curve {} has problematic y_max! y values: {:?}", + i, + &y[..y.len().min(10)] + ); + } + for y in y.iter_mut() { *y /= y_max; } @@ -189,8 +292,15 @@ pub fn violin( max = e; } } + eprintln!("[DIAG] violin: x min={}, max={}", min, max); + let mut dummy = [1.0]; let unit = formatter.scale_values(max, &mut dummy); + eprintln!( + "[DIAG] violin: unit={}, dummy after scale={:?}", + unit, dummy + ); + kdes.iter_mut().for_each(|&mut (_, ref mut xs, _)| { formatter.scale_values(max, xs); }); @@ -199,6 +309,16 @@ pub fn violin( x_range.start = 0.0; let y_range = -0.5..all_curves.len() as f64 - 0.5; + eprintln!( + "[DIAG] violin: x_range = {:?}, y_range = {:?}", + x_range, y_range + ); + eprintln!( + "[DIAG] violin: x_range.is_finite = {}, y_range.is_finite = {}", + x_range.start.is_finite() && x_range.end.is_finite(), + y_range.start.is_finite() && y_range.end.is_finite() + ); + let size = (960, 150 + (18 * all_curves.len() as u32)); let root_area = SVGBackend::new(&path, size) diff --git a/crates/divan_compat/examples/benches/time_scale.rs b/crates/divan_compat/examples/benches/time_scale.rs index c4c8baa7..ac214d0b 100644 --- a/crates/divan_compat/examples/benches/time_scale.rs +++ b/crates/divan_compat/examples/benches/time_scale.rs @@ -2,27 +2,32 @@ fn main() { divan::main(); } +fn busy_sleeping(duration: std::time::Duration) { + let start = std::time::Instant::now(); + while start.elapsed() < duration {} +} + #[divan::bench] fn sleep_1ns() { - std::thread::sleep(std::time::Duration::from_nanos(1)); + busy_sleeping(std::time::Duration::from_nanos(1)); } #[divan::bench] fn sleep_100ns() { - std::thread::sleep(std::time::Duration::from_nanos(100)); + busy_sleeping(std::time::Duration::from_nanos(100)); } #[divan::bench] fn sleep_1us() { - std::thread::sleep(std::time::Duration::from_micros(1)); + busy_sleeping(std::time::Duration::from_micros(1)); } #[divan::bench] fn sleep_100us() { - std::thread::sleep(std::time::Duration::from_micros(100)); + busy_sleeping(std::time::Duration::from_micros(100)); } #[divan::bench] fn sleep_1ms() { - std::thread::sleep(std::time::Duration::from_millis(1)); + busy_sleeping(std::time::Duration::from_millis(1)); }