Skip to content

Commit 79c5cc7

Browse files
pepeiborrapepeiborra
authored
Parallelize benchmark CI (#1320)
* [benchmark] command line config argument * [benchmark] Add a phony target for every example * [benchmark] add config to toggle heap profiling * [shake-bench] Phony rules for binaries * [CI] benchmark jobs tree * update mergify conditions
1 parent ec77814 commit 79c5cc7

File tree

8 files changed

+280
-93
lines changed

8 files changed

+280
-93
lines changed

‎.github/mergify.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ pull_request_rules:
55
method: squash
66
name: Automatically merge pull requests
77
conditions:
8-
- status-success=bench (8.10.3, ubuntu-latest)
8+
- status-success=bench-example (8.10.3, ubuntu-latest, Cabal-3.0.0.0)
9+
- status-success=bench-example (8.10.3, ubuntu-latest, lsp-types-1.0.0.1)
910

1011
- status-success=nix (default, ubuntu-latest)
1112
- status-success=nix (default, macOS-latest)

‎.github/workflows/bench.yml‎

Lines changed: 68 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Benchmark
22

33
on: [pull_request]
44
jobs:
5-
bench:
5+
bench-init:
66
runs-on: ${{ matrix.os }}
77

88
strategy:
@@ -38,20 +38,80 @@ jobs:
3838

3939
- name: Build
4040
shell: bash
41-
# Retry it three times to workaround compiler segfaults in windows
42-
run: cabal build ghcide:benchHist || cabal build ghcide:benchHist || cabal build ghcide:benchHist
41+
run: cabal build ghcide:benchHist
42+
43+
- name: Bench init
44+
shell: bash
45+
run: cabal bench ghcide:benchHist -j --benchmark-options="all-binaries"
46+
47+
# tar is required to preserve file permissions
48+
# compression speeds up upload/download nicely
49+
- name: tar workspace
50+
shell: bash
51+
run: tar -czf workspace.tar.gz * .git
52+
53+
- name: tar cabal
54+
run: |
55+
cd ~/.cabal
56+
tar -czf cabal.tar.gz *
57+
58+
- name: Upload workspace
59+
uses: actions/upload-artifact@v2
60+
with:
61+
name: workspace
62+
retention-days: 1
63+
path: workspace.tar.gz
64+
65+
- name: Upload .cabal
66+
uses: actions/upload-artifact@v2
67+
with:
68+
name: cabal-home
69+
retention-days: 1
70+
path: ~/.cabal/cabal.tar.gz
71+
72+
bench-example:
73+
needs: [bench-init]
74+
runs-on: ${{ matrix.os }}
75+
76+
strategy:
77+
fail-fast: false
78+
matrix:
79+
ghc: ['8.10.3']
80+
os: [ubuntu-latest]
81+
example: ['Cabal-3.0.0.0', 'lsp-types-1.0.0.1']
82+
83+
steps:
84+
- uses: haskell/actions/setup@v1
85+
with:
86+
ghc-version: ${{ matrix.ghc }}
87+
cabal-version: '3.2'
88+
enable-stack: false
89+
90+
- name: Download cabal home
91+
uses: actions/download-artifact@v2
92+
with:
93+
name: cabal-home
94+
path: .
95+
96+
- name: Download workspace
97+
uses: actions/download-artifact@v2
98+
with:
99+
name: workspace
100+
path: .
101+
102+
- name: untar
103+
run: |
104+
tar xzf workspace.tar.gz
105+
tar xzf cabal.tar.gz --directory ~/.cabal
43106
44107
- name: Bench
45108
shell: bash
46-
# run the tests without parallelism, otherwise tasty will attempt to run
47-
# all test cases simultaneously which causes way too many hls
48-
# instances to be spun up for the poor github actions runner to handle
49-
run: cabal bench ghcide:benchHist
109+
run: cabal bench ghcide:benchHist -j --benchmark-options="${{ matrix.example }}"
50110

51111
- name: Display results
52112
shell: bash
53113
run: |
54-
column -s, -t < ghcide/bench-results/results.csv | tee ghcide/bench-results/results.txt
114+
column -s, -t < ghcide/bench-results/unprofiled/${{ matrix.example }}/results.csv | tee ghcide/bench-results/unprofiled/${{ matrix.example }}/results.txt
55115
56116
- name: Archive benchmarking artifacts
57117
uses: actions/upload-artifact@v2

‎ghcide/.hlint.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@
9797
- flags:
9898
- default: false
9999
- {name: [-Wno-missing-signatures, -Wno-orphans, -Wno-overlapping-patterns, -Wno-incomplete-patterns, -Wno-missing-fields, -Wno-unused-matches]}
100-
- {name: [-Wno-dodgy-imports,-Wno-incomplete-uni-patterns], within: [Main, Development.IDE.GHC.Compat]}
100+
- {name: [-Wno-dodgy-imports,-Wno-incomplete-uni-patterns], within: [Main, Development.IDE.GHC.Compat, Development.Benchmark.Rules]}
101101
# - modules:
102102
# - {name: [Data.Set, Data.HashSet], as: Set} # if you import Data.Set qualified, it must be as 'Set'
103103
# - {name: Control.Arrow, within: []} # Certain modules are banned entirely

‎ghcide/bench/README.md‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,64 @@ performance analysis of ghcide:
66

77
- `exe/Main.hs` - a standalone benchmark runner. Run with `stack run ghcide-bench`
88
- `hist/Main.hs` - a Shake script for running the benchmark suite over a set of commits.
9-
- Run with `stack bench` or `cabal bench`,
9+
- Run with `stack bench ghcide` or `cabal bench ghcide`,
1010
- Requires a `ghcide-bench` binary in the PATH (usually provided by stack/cabal),
1111
- Calls `cabal` (or `stack`, configurable) internally to build the project,
12-
- Driven by the `config.yaml` configuration file.
12+
- Driven by the `bench/config.yaml` configuration file.
1313
By default it compares HEAD with "master"
1414

15-
Further details available in the config file and the module header comments.
15+
# Examples and experiments
16+
17+
The benchmark suites runs a set of experiments (hover, completion, edit, etc.)
18+
over all the defined examples (currently Cabal and lsp-types). Examples are defined
19+
in `ghcide/bench/config.yaml` whereas experiments are coded in `ghcide/bench/lib/Experiments.hs`.
20+
21+
# Phony targets
22+
23+
The Shake script supports a number of phony targets that allow running a subset of the benchmarks:
24+
25+
* all
26+
: runs all the examples, unprofiled
27+
28+
* profiled-all
29+
: runs all the examples with heap profiling, assuming `profilingInterval` is defined
30+
31+
* Cabal-3.0.0.0
32+
: runs the Cabal example, unprofiled
33+
34+
* profiled-Cabal-3.0.0.0
35+
: runs the Cabal example, with heap profiling
36+
37+
* etc
38+
39+
`--help` lists all the phony targets. Invoke it with:
40+
41+
cabal bench ghcide --benchmark-options="--help"
42+
43+
```
44+
Targets:
45+
- bench-results/binaries/*/commitid
46+
- bench-results/binaries/HEAD/ghcide
47+
- bench-results/binaries/HEAD/ghc.path
48+
- bench-results/binaries/*/ghcide
49+
- bench-results/binaries/*/ghc.path
50+
- bench-results/binaries/*/*.warmup
51+
- bench-results/*/*/*/*.csv
52+
- bench-results/*/*/*/*.gcStats.log
53+
- bench-results/*/*/*/*.output.log
54+
- bench-results/*/*/*/*.eventlog
55+
- bench-results/*/*/*/*.hp
56+
- bench-results/*/*/*/results.csv
57+
- bench-results/*/*/results.csv
58+
- bench-results/*/results.csv
59+
- bench-results/*/*/*/*.svg
60+
- bench-results/*/*/*/*.diff.svg
61+
- bench-results/*/*/*.svg
62+
- bench-results/*/*/*/*.heap.svg
63+
- Cabal-3.0.0.0
64+
- lsp-types-1.0.0.1
65+
- all
66+
- profiled-Cabal-3.0.0.0
67+
- profiled-lsp-types-1.0.0.1
68+
- profiled-all
69+
```

‎ghcide/bench/config.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,3 +67,7 @@ versions:
6767
# - ghcide-v0.7.3
6868
- upstream: origin/master
6969
- HEAD
70+
71+
# Heap profile interval in seconds (+RTS -i)
72+
# Comment out to disable heap profiling
73+
profileInterval: 1

‎ghcide/bench/hist/Main.hs‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,16 @@ import qualified Experiments.Types as E
5252
import GHC.Generics (Generic)
5353
import Numeric.Natural (Natural)
5454
import Development.Shake.Classes
55+
import System.Console.GetOpt
56+
import Data.Maybe
57+
import Control.Monad.Extra
5558

5659

57-
config :: FilePath
58-
config = "bench/config.yaml"
60+
configPath :: FilePath
61+
configPath = "bench/config.yaml"
62+
63+
configOpt :: OptDescr (Either String FilePath)
64+
configOpt = Option [] ["config"] (ReqArg Right configPath) "config file"
5965

6066
-- | Read the config without dependency
6167
readConfigIO :: FilePath -> IO (Config BuildSystem)
@@ -65,17 +71,17 @@ instance IsExample Example where getExampleName = E.getExampleName
6571
type instance RuleResult GetExample = Maybe Example
6672
type instance RuleResult GetExamples = [Example]
6773

74+
shakeOpts :: ShakeOptions
75+
shakeOpts =
76+
shakeOptions{shakeChange = ChangeModtimeAndDigestInput, shakeThreads = 0}
77+
6878
main :: IO ()
69-
main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigestInput, shakeThreads = 0} $ do
70-
createBuildSystem $ \resource -> do
71-
configStatic <- liftIO $ readConfigIO config
72-
let build = outputFolder configStatic
73-
buildRules build ghcideBuildRules
74-
benchRules build resource (MkBenchRules (askOracle $ GetSamples ()) benchGhcide "ghcide")
75-
csvRules build
76-
svgRules build
77-
heapProfileRules build
78-
action $ allTargets build
79+
main = shakeArgsWith shakeOpts [configOpt] $ \configs wants -> pure $ Just $ do
80+
let config = fromMaybe configPath $ listToMaybe configs
81+
_configStatic <- createBuildSystem config
82+
case wants of
83+
[] -> want ["all"]
84+
_ -> want wants
7985

8086
ghcideBuildRules :: MkBuildRules BuildSystem
8187
ghcideBuildRules = MkBuildRules findGhcForBuildSystem "ghcide" buildGhcide
@@ -89,13 +95,14 @@ data Config buildSystem = Config
8995
versions :: [GitCommit],
9096
-- | Output folder ('foo' works, 'foo/bar' does not)
9197
outputFolder :: String,
92-
buildTool :: buildSystem
98+
buildTool :: buildSystem,
99+
profileInterval :: Maybe Double
93100
}
94101
deriving (Generic, Show)
95102
deriving anyclass (FromJSON)
96103

97-
createBuildSystem :: (Resource -> Rules a) -> Rules a
98-
createBuildSystem userRules = do
104+
createBuildSystem :: FilePath -> Rules (Config BuildSystem )
105+
createBuildSystem config = do
99106
readConfig <- newCache $ \fp -> need [fp] >> liftIO (readConfigIO fp)
100107

101108
_ <- addOracle $ \GetExperiments {} -> experiments <$> readConfig config
@@ -105,9 +112,20 @@ createBuildSystem userRules = do
105112
_ <- addOracle $ \GetBuildSystem {} -> buildTool <$> readConfig config
106113
_ <- addOracle $ \GetSamples{} -> samples <$> readConfig config
107114

108-
benchResource <- newResource "ghcide-bench" 1
115+
configStatic <- liftIO $ readConfigIO config
116+
let build = outputFolder configStatic
117+
118+
buildRules build ghcideBuildRules
119+
benchRules build (MkBenchRules (askOracle $ GetSamples ()) benchGhcide "ghcide")
120+
csvRules build
121+
svgRules build
122+
heapProfileRules build
123+
phonyRules "" "ghcide" NoProfiling build (examples configStatic)
124+
125+
whenJust (profileInterval configStatic) $ \i -> do
126+
phonyRules "profiled-" "ghcide" (CheapHeapProfiling i) build (examples configStatic)
109127

110-
userRules benchResource
128+
return configStatic
111129

112130
newtype GetSamples = GetSamples () deriving newtype (Binary, Eq, Hashable, NFData, Show)
113131
type instance RuleResult GetSamples = Natural

‎ghcide/ghcide.cabal‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,9 @@ benchmark benchHist
246246
base == 4.*,
247247
shake-bench == 0.1.*,
248248
directory,
249+
extra,
249250
filepath,
251+
optparse-applicative,
250252
shake,
251253
text,
252254
yaml

0 commit comments

Comments
 (0)