Partition tests into multiple CI jobs to reduce runtime and memory pressure (#2253)

Author: odow

.github/workflows/ci.yml

@@ -8,7 +8,7 @@ on:
     types: [opened, synchronize, reopened]
 jobs:
   test:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.moi_test_modules }} - ${{ github.event_name }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -19,18 +19,43 @@ jobs:
           - version: 'nightly'
             os: ubuntu-latest
             arch: x64
+            moi_test_modules: 'General;Nonlinear;Bridges;FileFormats'
+          - version: 'nightly'
+            os: ubuntu-latest
+            arch: x64
+            moi_test_modules: 'Test;Utilities;Benchmarks'
+          - version: '1'
+            os: ubuntu-latest
+            arch: x64
+            moi_test_modules: 'General;Nonlinear;Bridges;FileFormats'
           - version: '1'
             os: ubuntu-latest
             arch: x64
+            moi_test_modules: 'Test;Utilities;Benchmarks'
+          - version: '1'
+            os: windows-latest
+            arch: x64
+            moi_test_modules: 'General;Nonlinear;Bridges;FileFormats'
           - version: '1'
             os: windows-latest
             arch: x64
+            moi_test_modules: 'Test;Utilities;Benchmarks'
+          - version: '1.6'
+            os: ubuntu-latest
+            arch: x64
+            moi_test_modules: 'General;Nonlinear;Bridges;FileFormats'
           - version: '1.6'
             os: ubuntu-latest
             arch: x64
+            moi_test_modules: 'Test;Utilities;Benchmarks'
           - version: '1'
             os: ubuntu-latest
             arch: x86
+            moi_test_modules: 'General;Nonlinear;Bridges;FileFormats'
+          - version: '1'
+            os: ubuntu-latest
+            arch: x86
+            moi_test_modules: 'Test;Utilities;Benchmarks'
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1
@@ -49,6 +74,8 @@ jobs:
             ${{ runner.os }}-
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+        env:
+            MOI_TEST_MODULES: ${{ matrix.moi_test_modules }}
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:

test/runtests.jl

@@ -4,30 +4,57 @@
 # Use of this source code is governed by an MIT-style license that can be found
 # in the LICENSE.md file or at https://opensource.org/licenses/MIT.
 
+# To try and speed up the tests, MOI uses a `MOI_TEST_MODULES` environment
+# variable. This environment variable may be missing, or it may be a subset of
+# the following, concatenated with `;` as a separator:
+#
+#  * General
+#  * Benchmarks
+#  * Bridges
+#  * FileFormats
+#  * Nonlinear
+#  * Test
+#  * Utilities
+#
+# If present, the tests run only those submodules defined above. `General` is
+# not a submodule, but it runs all of the top-level tests in MOI.
+
 using Test
 
 # This file gets called first. If it doesn't crash, all is well.
 include("issue980.jl")
 
 import MathOptInterface as MOI
-@test isempty(Test.detect_ambiguities(MOI; recursive = true))
 
-for file in readdir(@__DIR__)
-    if file in ["issue980.jl", "dummy.jl", "hygiene.jl", "runtests.jl"]
-        continue
-    elseif !endswith(file, ".jl")
-        continue
-    end
-    @testset "$(file)" begin
-        include(file)
+MODULES_TO_TEST = get(
+    ENV,
+    "MOI_TEST_MODULES",
+    "General;Benchmarks;Bridges;FileFormats;Nonlinear;Test;Utilities",
+)
+
+if occursin("General", MODULES_TO_TEST)
+    @test isempty(Test.detect_ambiguities(MOI; recursive = true))
+    for file in readdir(@__DIR__)
+        if file in ["issue980.jl", "dummy.jl", "hygiene.jl", "runtests.jl"]
+            continue
+        elseif !endswith(file, ".jl")
+            continue
+        end
+        @testset "$(file)" begin
+            include(file)
+        end
     end
 end
 
-for submodule in
-    ["Nonlinear", "Bridges", "FileFormats", "Test", "Utilities", "Benchmarks"]
+for submodule in split(MODULES_TO_TEST, ";")
+    if submodule == "General"
+        continue
+    end
     include("$(submodule)/$(submodule).jl")
     GC.gc()  # Force GC run here to reduce memory pressure
 end
 
-# Test hygiene of @model macro
-include("hygiene.jl")
+if occursin("General", MODULES_TO_TEST)
+    # Test hygiene of @model macro
+    include("hygiene.jl")
+end