Kind ctestInfo support beta

[z-aki]

code-stdin-ztU.json.txt

It's not documented but ctest --show-only=json-v1 gives this output and it's hinted to be supported at https://gitlab.kitware.com/cmake/cmake/-/merge_requests/2499#note_476147

[z-aki]

FYI i'm working on it

[madebr]

I saw your 2nd message too late..
I have a poc at c77bdc7

[z-aki]

(: thanks!

I'll post it anyway.. added a test

This seems to be missing "command" key

[z-aki]

Here's a diff too

My diff

diff --git a/cmake_file_api/cmake.py b/cmake_file_api/cmake.py
index 610ae9a..080703c 100644
--- a/cmake_file_api/cmake.py
+++ b/cmake_file_api/cmake.py
@@ -73,6 +73,12 @@ class CMakeProject(object):
             args.append(".")
         subprocess.check_call(args, cwd=str(self._build_path), stdout=stdout)
 
+    def build_target(self, target: str, quiet=False):
+        stdout = subprocess.DEVNULL if quiet else None
+        args = [str(self._cmake), "--build",
+                str(self._build_path), "--target", target]
+        subprocess.check_call(args, cwd=str(self._build_path), stdout=stdout)
+
     @property
     def cmake_file_api(self):
         return REPLY_API[self._api_version](self._build_path)
diff --git a/cmake_file_api/kinds/ctestInfo/__init__.py b/cmake_file_api/kinds/ctestInfo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cmake_file_api/kinds/ctestInfo/api.py b/cmake_file_api/kinds/ctestInfo/api.py
new file mode 100644
index 0000000..c74a90a
--- /dev/null
+++ b/cmake_file_api/kinds/ctestInfo/api.py
@@ -0,0 +1,5 @@
+from .v1 import CTestInfoV1
+
+CTEST_INFO_API = {
+    1: CTestInfoV1,
+}
\ No newline at end of file
diff --git a/cmake_file_api/kinds/ctestInfo/v1.py b/cmake_file_api/kinds/ctestInfo/v1.py
new file mode 100644
index 0000000..a0a9fe7
--- /dev/null
+++ b/cmake_file_api/kinds/ctestInfo/v1.py
@@ -0,0 +1,73 @@
+import json
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from cmake_file_api.kinds.common import VersionMajorMinor
+from cmake_file_api.kinds.kind import ObjectKind
+
+
+class CTestInfoTestProperties(object):
+    __slots__ = ("name", "value")
+
+    def __init__(self, name: str, value: str):
+        self.name = name
+        self.value = value
+
+    @classmethod
+    def from_dict(cls, dikt: Dict) -> "CTestInfoTestProperties":
+        name = dikt["name"]
+        value = dikt["value"]
+        return cls(name, value)
+
+    def __repr__(self) -> str:
+        return "{}(name='{}', value='{}')".format(
+            type(self).__name__,
+            self.name,
+            self.value
+        )
+
+    def isEnvironment(self) -> bool:
+        return self.name == "ENVIRONMENT"
+
+class CTestInfoTest(object):
+    __slots__ = ("name", "command", "properties")
+
+    def __init__(self, name: str, properties: List[CTestInfoTestProperties], command: List[str]):
+        self.name = name
+        self.properties = properties
+        self.command = command
+
+    @classmethod
+    def from_dict(cls, dikt: Dict) -> "CTestInfoTest":
+        name = dikt["name"]
+        properties = [CTestInfoTestProperties.from_dict(p) for p in dikt["properties"]]
+        command = dikt["command"]
+        return cls(name, properties, command)
+
+class CTestInfoV1(object):
+    KIND = ObjectKind.CTEST_INFO
+
+    __slots__ = {"version", "tests"}
+
+    def __init__(self, version: VersionMajorMinor, tests: List[CTestInfoTest]):
+        self.version = version
+        self.tests = tests
+
+    @classmethod
+    def from_dict(cls, dikt: Dict) -> "CTestInfoV1":
+        version = VersionMajorMinor.from_dict(dikt["version"])
+        tests = [CTestInfoTest.from_dict(t) for t in dikt["tests"]]
+        return cls(version, tests)
+
+    @classmethod
+    def from_path(cls, path: Path, reply_path: Path) -> "CTestInfoV1":
+        with (reply_path).open("r") as f:
+            return cls.from_dict(json.load(f))
+
+
+    def __repr__(self) -> str:
+        return "{}(version={}, tests={})".format(
+            type(self).__name__,
+            self.version,
+            self.tests
+        )
diff --git a/cmake_file_api/kinds/kind.py b/cmake_file_api/kinds/kind.py
index 640826b..51f3f42 100644
--- a/cmake_file_api/kinds/kind.py
+++ b/cmake_file_api/kinds/kind.py
@@ -6,4 +6,5 @@ class ObjectKind(enum.Enum):
     CMAKEFILES = "cmakeFiles"
     CODEMODEL = "codemodel"
     CONFIGURELOG = "configureLog"
+    CTEST_INFO = "ctestInfo"
     TOOLCHAINS = "toolchains"
diff --git a/tests/resources/ctestInfoOutput1.json b/tests/resources/ctestInfoOutput1.json
new file mode 100644
index 0000000..d53309e
--- /dev/null
+++ b/tests/resources/ctestInfoOutput1.json
@@ -0,0 +1,113 @@
+{
+    "backtraceGraph": {
+        "commands": [
+            "add_test",
+            "add_new_test"
+        ],
+        "files": [
+            "/Foobar/CMakeLists.txt"
+        ],
+        "nodes": [
+            {
+                "file": 0
+            },
+            {
+                "command": 0,
+                "file": 0,
+                "line": 6,
+                "parent": 0
+            },
+            {
+                "file": 0
+            },
+            {
+                "command": 1,
+                "file": 0,
+                "line": 11,
+                "parent": 2
+            },
+            {
+                "command": 0,
+                "file": 0,
+                "line": 8,
+                "parent": 3
+            },
+            {
+                "file": 0
+            },
+            {
+                "command": 1,
+                "file": 0,
+                "line": 12,
+                "parent": 5
+            },
+            {
+                "command": 0,
+                "file": 0,
+                "line": 8,
+                "parent": 6
+            }
+        ]
+    },
+    "kind": "ctestInfo",
+    "tests": [
+        {
+            "backtrace": 1,
+            "command": [
+                "/Foobar/build0/MagicMath",
+                "4"
+            ],
+            "name": "Test_runs",
+            "properties": [
+                {
+                    "name": "WORKING_DIRECTORY",
+                    "value": "/Foobar/build0"
+                }
+            ]
+        },
+        {
+            "backtrace": 4,
+            "command": [
+                "/Foobar/build0/MagicMath",
+                "9"
+            ],
+            "name": "Test_9_3",
+            "properties": [
+                {
+                    "name": "PASS_REGULAR_EXPRESSION",
+                    "value": [
+                        "3"
+                    ]
+                },
+                {
+                    "name": "WORKING_DIRECTORY",
+                    "value": "/Foobar/build0"
+                }
+            ]
+        },
+        {
+            "backtrace": 7,
+            "command": [
+                "/Foobar/build0/MagicMath",
+                "25"
+            ],
+            "name": "Test_25_5",
+            "properties": [
+                {
+                    "name": "PASS_REGULAR_EXPRESSION",
+                    "value": [
+                        "5"
+                    ]
+                },
+                {
+                    "name": "WORKING_DIRECTORY",
+                    "value": "/Foobar/build0"
+                }
+            ]
+        }
+    ],
+    "version": {
+        "major": 1,
+        "minor": 0
+    }
+}
\ No newline at end of file
diff --git a/tests/test_regression.py b/tests/test_regression.py
index bc3697a..616a9c7 100644
--- a/tests/test_regression.py
+++ b/tests/test_regression.py
@@ -9,6 +9,7 @@ import pytest
 from cmake_file_api.cmake import CMakeProject
 from cmake_file_api.kinds.kind import ObjectKind
 from cmake_file_api.kinds.codemodel.api import CODEMODEL_API
+from cmake_file_api.kinds.ctestInfo.api import CTEST_INFO_API
 
 
 @functools.lru_cache(1)  # FIXME: CPython 3.9 provides `functools.cache`
@@ -22,6 +23,14 @@ def cmake_version():
 
 CMAKE_SUPPORTS_TOOLCHAINS_V1 = cmake_version() >= (3, 20)
 
+def load_json_directly(path):
+    import json
+    from pathlib import Path
+    return json.loads(Path(path).read_text())
+
+def load_json_ctest(path):
+    dikt = load_json_directly(path)
+    return CTEST_INFO_API[1].from_dict(dikt)
 
 @pytest.fixture
 def build_tree(tmp_path_factory):
@@ -47,6 +56,37 @@ def simple_cxx_project(build_tree):
     return build_tree
 
 
+@pytest.fixture
+def simple_cxx_project_with_tests(build_tree):
+    (build_tree.source / "CMakeLists.txt").write_text(textwrap.dedent("""\
+        cmake_minimum_required(VERSION 3.0)
+        project(demoproject)
+        add_executable(MagicMath main.cpp)
+        enable_testing()
+        install(TARGETS MagicMath DESTINATION bin)
+        add_test(NAME Test_runs COMMAND MagicMath 4)
+        function(add_new_test name argToMagic logExpected)
+            add_test(NAME ${name} COMMAND MagicMath ${argToMagic})
+            set_tests_properties(${name} PROPERTIES PASS_REGULAR_EXPRESSION ${logExpected})
+        endfunction()
+        add_new_test(Test_9_3 "9" "3")
+        add_new_test(Test_25_5 "25" "5")
+        """))
+    (build_tree.source / "main.cpp").write_text(textwrap.dedent(r"""\
+        #include <iostream>
+        #include <string>
+        int main(int argc, char *argv[]) {
+            if (argc < 2) {
+                std::cout << "Usage: " << argv[0] << " number" << std::endl;
+                return 1;
+            }
+            int inputValue = std::stoi(argv[1]);
+            std::cout << "The magic number is " << sqrt(inputValue) << std::endl;
+            return 0;
+        }"""))
+    return build_tree
+
+
 @pytest.fixture
 def complex_cxx_project(build_tree):
     (build_tree.source / "CMakeLists.txt").write_text(textwrap.dedent("""\
@@ -199,3 +239,57 @@ def test_toolchain_kind_cxx(complex_cxx_project, capsys):
     assert isinstance(kind_obj.version, VersionMajorMinor)
     assert kind_obj.version.major == 1
     assert "CXX" in tuple(toolchain.language for toolchain in kind_obj.toolchains)
+
+
+def test_ctest_info(simple_cxx_project_with_tests, capsys):
+    project = CMakeProject(simple_cxx_project_with_tests.build,
+                           simple_cxx_project_with_tests.source, api_version=1)
+    project.cmake_file_api.instrument_all()
+    project.reconfigure(quiet=True)
+    project.build_target("MagicMath", quiet=True)
+    import subprocess
+    import json
+    from cmake_file_api.kinds.ctestInfo.v1 import CTestInfoV1
+
+    json_from_ctest = json.loads(subprocess.check_output(
+        ["ctest", "--show-only=json-v1"], cwd=simple_cxx_project_with_tests.build))
+    print(json_from_ctest)
+    print(simple_cxx_project_with_tests.build)
+    kind_obj = CTestInfoV1.from_dict(json_from_ctest)
+
+    from cmake_file_api.kinds.common import VersionMajorMinor
+
+    assert isinstance(kind_obj, CTestInfoV1)
+    assert isinstance(kind_obj.version, VersionMajorMinor)
+    assert kind_obj.version.major == 1
+
+
+def test_ctest_from_json(capsys):
+    from cmake_file_api.kinds.ctestInfo.v1 import CTestInfoV1
+    from cmake_file_api.kinds.common import VersionMajorMinor
+
+    kind_obj: CTestInfoV1 = load_json_ctest(
+        "tests/resources/ctestInfoOutput1.json")
+
+    assert isinstance(kind_obj, CTestInfoV1)
+    assert isinstance(kind_obj.version, VersionMajorMinor)
+    assert kind_obj.version.major == 1
+    assert len(kind_obj.tests) == 3
+    assert kind_obj.tests[0].name == "Test_runs"
+    assert len(kind_obj.tests[0].properties) == 1
+    assert kind_obj.tests[0].command == [
+        "/Foobar/build0/MagicMath",
+        "4"
+    ]
+    assert kind_obj.tests[1].name == "Test_9_3"
+    assert len(kind_obj.tests[1].properties) == 2
+    assert kind_obj.tests[1].command == [
+        "/Foobar/build0/MagicMath",
+        "9"
+    ]
+    assert kind_obj.tests[2].name == "Test_25_5"
+    assert len(kind_obj.tests[2].properties) == 2
+    assert kind_obj.tests[2].command == [
+        "/Foobar/build0/MagicMath",
+        "25"
+    ]

[madebr]

I think the command key is optional.
Run the following in the examples directory of my poc:

cd $PWD && cmake ../project && python -c 'import subprocess;import json; import pprint;pprint.pprint(json.loads(subprocess.check_output(["ctest","--show-only=json-v1"],text=True))["tests"])'

It prints:

[{'backtrace': 1,
  'name': 'exe_noargs',
  'properties': [{'name': 'ENVIRONMENT',
                  'value': ['FOO=BAR', 'CAFEBABE=DEADBABE']},
                 {'name': 'TIMEOUT', 'value': 2.0},
                 {'name': 'WORKING_DIRECTORY',
                  'value': '/home/maarten/programming/python-cmake-file-api/example/build/exe'}]},
 {'backtrace': 3,
  'name': 'exe_1arg',
  'properties': [{'name': 'ENVIRONMENT',
                  'value': ['FOO=BAR', 'CAFEBABE=DEADBABE']},
                 {'name': 'TIMEOUT', 'value': 2.0},
                 {'name': 'WORKING_DIRECTORY',
                  'value': '/home/maarten/programming/python-cmake-file-api/example/build/exe'}]},
 {'backtrace': 5,
  'name': 'exe_2args',
  'properties': [{'name': 'ENVIRONMENT',
                  'value': ['FOO=BAR', 'CAFEBABE=DEADBABE']},
                 {'name': 'TIMEOUT', 'value': 2.0},
                 {'name': 'WORKING_DIRECTORY',
                  'value': '/home/maarten/programming/python-cmake-file-api/example/project'}]}]

Thanks for the tests, I'm going to incorporate them!

[z-aki]

    add_library(lib1_install lib1.cpp)
    install(TARGETS lib1_install)
    add_test(NAME lib1_install COMMAND lib1_install)

test and library have the same name.. and COMMAND is also a library, how would it run on CLI ? (:
Good values are executables and generator expressions https://cmake.org/cmake/help/latest/command/add_test.html?highlight=add_test

[z-aki]

I think the command key is optional.

Maybe command = dikt.get("command", []) would work

[madebr]

Looks like the command has a quirk: you need to build the executables first.

$ cat CMakeLists.txt 
cmake_minimum_required(VERSION 3.20)
project(ss C)

enable_testing()

file(WRITE run.c [[
int main(int argc, char *argv[]) {
  return 0;
}
]])

add_executable(run run.c)
add_test(run run)
$ cmake -S . -B build
-- The C compiler identification is GNU 12.2.1
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /usr/lib64/ccache/cc - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Configuring done
-- Generating done
-- Build files have been written to: /tmp/ss/build
[maarten@fedora ss]$ cd build
[maarten@fedora build]$ ctest --show-only=json-v1 | jq .tests
[
  {
    "backtrace": 1,
    "name": "run",
    "properties": [
      {
        "name": "WORKING_DIRECTORY",
        "value": "/tmp/ss/build"
      }
    ]
  }
]
$ cmake --build .
[ 50%] Building C object CMakeFiles/run.dir/run.c.o
[100%] Linking C executable run
[100%] Built target run
$ ctest --show-only=json-v1 | jq .tests
[
  {
    "backtrace": 1,
    "command": [
      "/tmp/ss/build/run"
    ],
    "name": "run",
    "properties": [
      {
        "name": "WORKING_DIRECTORY",
        "value": "/tmp/ss/build"
      }
    ]
  }
]

I'm going to do command = dikt.get("command") and leave it None if not present.

[madebr]

I created https://gitlab.kitware.com/cmake/cmake/-/issues/24668

[z-aki]

Not a limitation of this library: When using gtest test discovery, executables are built and then they provide what test they contain to ctest. So caller of this library will have to run it again to get latest output