Merge pull request #6 from divmadan/polish-parse

Introduce clang_utils.py, compilation_database.py and related changes

Author: kunaltyagi

.github/workflows/pytest.yml

@@ -6,7 +6,7 @@ on: [push, pull_request]
 jobs:
   Pytest:
     # The type of runner that the job will run on
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
@@ -34,12 +34,12 @@ jobs:
       - name: Add llvm keys
         run: |
           wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-          echo 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list
-          echo 'deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list
+          echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-12 main' | sudo tee -a /etc/apt/sources.list
+          echo 'deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-12 main' | sudo tee -a /etc/apt/sources.list
       - name: Install libclang and its python bindings
         run: |
           sudo apt-get update
-          sudo apt-get install -y libclang-11-dev python3-clang-11
+          sudo apt-get install -y libclang-12-dev python3-clang-12
 
       # Add dist-package to path to enable apt installed python3-clang import
       - name: Add dist-packages to PYTHONPATH

bindings/python/scripts/clang_utils.py

@@ -0,0 +1,120 @@
+import inspect
+import clang.cindex as clang
+
+
+def getmembers_static(object, predicate=None):
+    """
+    Return all members of an object as (name, value) pairs sorted by name via `getattr_static`.
+    Optionally, only return members that satisfy a given predicate.
+
+
+    - A static version of `get_members` function at:
+      https://github.com/python/cpython/blob/3.9/Lib/inspect.py#L326-L368
+      https://github.com/python/cpython/blob/14ba761078b5ae83519e34d66ab883743912c45b/Lib/inspect.py#L444-L486
+        - `getmembers` function (from the inspect module) triggers execution instead of doing static analysis.
+        - This leads to errors, particularly on properties of classes in cindex.py, which causes segmentation errors or raises an Exception if a particular condition is not satisfied.
+        - To curb this, we fetch the members statically. We define a custom function based on the one in the inspect module.
+    """
+
+    results = []
+    names = dir(object)
+    # :dd any DynamicClassAttributes to the list of names if object is a class;
+    # this may result in duplicate entries if, for example, a virtual
+    # attribute with the same name as a DynamicClassAttribute exists
+    try:
+        base_members = filter(
+            lambda k, v: isinstance(v, types.DynamicClassAttribute),
+            object.__bases__.__dict__.items(),
+        )
+        names.extend(base_members)
+    except AttributeError:
+        pass
+    for key in names:
+        value = inspect.getattr_static(object, key)
+        if not predicate or predicate(value):
+            results.append((key, value))
+    results.sort(key=lambda pair: pair[0])
+    return results
+
+
+class ClangUtils:
+    """
+    Clang's cindex class utilities.
+
+    Supports the following objects:
+        CursorKind:
+          https://github.com/llvm/llvm-project/blob/release/12.x/clang/bindings/python/clang/cindex.py#L657
+          https://github.com/llvm/llvm-project/blob/1acd9a1a29ac30044ecefb6613485d5d168f66ca/clang/bindings/python/clang/cindex.py#L657
+            - A CursorKind describes the kind of entity that a cursor points to.
+        Cursor:
+          https://github.com/llvm/llvm-project/blob/release/12.x/clang/bindings/python/clang/cindex.py#L1415
+          https://github.com/llvm/llvm-project/blob/1acd9a1a29ac30044ecefb6613485d5d168f66ca/clang/bindings/python/clang/cindex.py#L1415
+            - The Cursor class represents a reference to an element within the AST. It acts as a kind of iterator.
+        Type:
+          https://github.com/llvm/llvm-project/blob/release/12.x/clang/bindings/python/clang/cindex.py#L2180
+          https://github.com/llvm/llvm-project/blob/1acd9a1a29ac30044ecefb6613485d5d168f66ca/clang/bindings/python/clang/cindex.py#L2180
+            - The Type class represents the type of an element in the abstract syntax tree.
+    """
+
+    def __init__(self, object):
+        if not (
+            isinstance(object, clang.CursorKind)
+            or isinstance(object, clang.Cursor)
+            or isinstance(object, clang.Type)
+        ):
+            raise NotImplementedError(f"Not implemented for {object}")
+
+        self.check_functions_dict = {}
+        self.get_functions_dict = {}
+        self.properties_dict = {}
+
+        # A list to ignore the functions/properties that causes segmentation errors.
+        ignore_list = [
+            "mangled_name",
+            "get_address_space",
+            "get_typedef_name",
+            "tls_kind",
+        ]
+
+        # populate dicts
+        valid_entries = filter(
+            lambda entry: entry[0] not in ignore_list, getmembers_static(object)
+        )
+        for name, func in valid_entries:
+            if inspect.isfunction(func):  # if function
+                try:  # cindex.py's functions raise exceptions internally
+                    if name.startswith("is_"):
+                        self.check_functions_dict[name] = func(object)
+                    if name.startswith("get_"):
+                        self.get_functions_dict[name] = func(object)
+                except:
+                    continue
+            elif isinstance(func, property):  # else, property
+                try:  # cindex.py's property functions raise exceptions internally
+                    self.properties_dict[name] = getattr(object, name)
+                except:
+                    continue
+
+    def get_check_functions_dict(self):
+        """
+        Returns: `check_functions_dict`:
+            - functions that begin with "is_" i.e., checking functions
+            - {function_name, function_result}
+        """
+        return self.check_functions_dict
+
+    def get_get_functions_dict(self):
+        """
+        Returns: `get_functions_dict`:
+            - functions that begin with "get_" i.e., getter functions
+            - {function_name, function_result}
+        """
+        return self.get_functions_dict
+
+    def get_properties_dict(self):
+        """
+        Returns: properties_dict
+            - Properties
+            - {property_name, property}
+        """
+        return self.properties_dict

bindings/python/scripts/compilation_database.py

@@ -0,0 +1,40 @@
+import clang.cindex as clang
+
+
+class CompilationDatabase:
+    """
+    Build a compilation database from a given directory
+    """
+
+    def __init__(self, compilation_database_path):
+        self.compilation_database = clang.CompilationDatabase.fromDirectory(
+            buildDir=compilation_database_path
+        )
+
+    def get_compilation_arguments(self, filename=None):
+        """
+        Returns the compilation commands extracted from the compilation database
+
+        Parameters:
+            - compilation_database_path: The path to `compile_commands.json`
+            - filename (optional): To get compilaton commands of a file
+
+        Returns:
+            - compilation_arguments (dict): {filename: compiler arguments}
+        """
+
+        if filename:
+            # Get compilation commands from the compilation database for the given file
+            compilation_commands = self.compilation_database.getCompileCommands(
+                filename=filename
+            )
+        else:
+            # Get all compilation commands from the compilation database
+            compilation_commands = self.compilation_database.getAllCompileCommands()
+
+        # {file: compiler arguments}
+        compilation_arguments = {
+            command.filename: list(command.arguments)[1:-1]
+            for command in compilation_commands
+        }
+        return compilation_arguments

bindings/python/scripts/generate.py

@@ -148,10 +148,14 @@ def get_fields_from_anonymous(item: dict) -> list:
         fields = []
         for sub_item in item["members"]:
             # base condition
-            if sub_item["kind"] == "FIELD_DECL":
+            if sub_item["cursor_kind"]["name"] == "FIELD_DECL":
                 fields.append(sub_item)
             # recurse
-            elif sub_item["kind"] in ("ANONYMOUS_UNION_DECL", "ANONYMOUS_STRUCT_DECL"):
+            # @TODO Fix this, `ANONYMOUS_kind` was removed, now test via `is_anonymous`
+            elif sub_item["cursor_kind"]["name"] in (
+                "ANONYMOUS_UNION_DECL",
+                "ANONYMOUS_STRUCT_DECL",
+            ):
                 fields += bind.get_fields_from_anonymous(item=sub_item)
         return fields
 
@@ -173,8 +177,8 @@ def handle_node(self, item: dict) -> None:
         """
 
         self.item = item
-        self.kind = self.item["kind"]
-        self.name = self.item["name"]
+        self.kind = self.item["cursor_kind"]["name"]
+        self.name = self.item["cursor"]["spelling"]
         self.members = self.item["members"]
         self.depth = self.item["depth"]
 
@@ -218,17 +222,21 @@ def handle_struct_decl(self) -> None:
         template_class_name = None
         template_class_name_python = None
         for sub_item in self.members:
-            if sub_item["kind"] == "TYPE_REF":
+            if sub_item["cursor_kind"]["name"] == "TYPE_REF":
                 # TODO: Will this case only apply to templates?
                 # @TODO: Make more robust
-                type_ref = sub_item["name"].replace("struct ", "").replace("pcl::", "")
+                type_ref = (
+                    sub_item["cursor"]["spelling"]
+                    .replace("struct ", "")
+                    .replace("pcl::", "")
+                )
                 template_class_name = f"{self.name}<{type_ref}>"
                 template_class_name_python = f"{self.name}_{type_ref}"
 
         base_class_list = [
-            sub_item["name"]
+            sub_item["cursor"]["spelling"]
             for sub_item in self.members
-            if sub_item["kind"] == "CXX_BASE_SPECIFIER"
+            if sub_item["cursor_kind"]["name"] == "CXX_BASE_SPECIFIER"
         ]
 
         base_class_list_string = [
@@ -253,35 +261,35 @@ def handle_struct_decl(self) -> None:
         for sub_item in self.members:
             fields = self.get_fields_from_anonymous(sub_item)
             for field in fields:
-                if field["element_type"] == "ConstantArray":
+                if field["type"]["kind"] == "ConstantArray":
                     # TODO: FIX: readwrite, not readonly
                     self._linelist.append(
-                        f'.def_property_readonly("{field["name"]}", []({self.name}& obj) {{return obj.{field["name"]}; }})'  # float[ ' + f'obj.{sub_item["name"]}' + '.size()];} )'
+                        f'.def_property_readonly("{field["cursor"]["spelling"]}", []({self.name}& obj) {{return obj.{field["cursor"]["spelling"]}; }})'  # float[ ' + f'obj.{sub_item["cursor"]["spelling"]}' + '.size()];} )'
                     )
                 else:
                     self._linelist.append(
-                        f'.def_readwrite("{field["name"]}", &{self.name}::{field["name"]})'
+                        f'.def_readwrite("{field["cursor"]["spelling"]}", &{self.name}::{field["cursor"]["spelling"]})'
                     )
 
         for sub_item in self.members:
 
             # handle field declarations
-            if sub_item["kind"] == "FIELD_DECL":
-                if sub_item["element_type"] == "ConstantArray":
+            if sub_item["cursor_kind"]["name"] == "FIELD_DECL":
+                if sub_item["type"]["kind"] == "ConstantArray":
                     self._linelist.append(
-                        f'.def_property_readonly("{sub_item["name"]}", []({self.name}& obj) {{return obj.{sub_item["name"]}; }})'  # float[ ' + f'obj.{sub_item["name"]}' + '.size()];} )'
+                        f'.def_property_readonly("{sub_item["cursor"]["spelling"]}", []({self.name}& obj) {{return obj.{sub_item["cursor"]["spelling"]}; }})'  # float[ ' + f'obj.{sub_item["cursor"]["spelling"]}' + '.size()];} )'
                     )
                 else:
                     self._linelist.append(
-                        f'.def_readwrite("{sub_item["name"]}", &{self.name}::{sub_item["name"]})'
+                        f'.def_readwrite("{sub_item["cursor"]["spelling"]}", &{self.name}::{sub_item["cursor"]["spelling"]})'
                     )
 
             # handle class methods
-            elif sub_item["kind"] == "CXX_METHOD":
+            elif sub_item["cursor_kind"]["name"] == "CXX_METHOD":
                 # TODO: Add template args, currently blank
-                if sub_item["name"] not in ("PCL_DEPRECATED"):
+                if sub_item["cursor"]["spelling"] not in ("PCL_DEPRECATED"):
                     self._linelist.append(
-                        f'.def("{sub_item["name"]}", py::overload_cast<>(&{self.name}::{sub_item["name"]}))'
+                        f'.def("{sub_item["cursor"]["spelling"]}", py::overload_cast<>(&{self.name}::{sub_item["cursor"]["spelling"]}))'
                     )
 
     def handle_function(self) -> None:
@@ -293,8 +301,8 @@ def handle_function(self) -> None:
         parameter_type_list = []
         details = self._state_stack[-1]
         for sub_item in self.members:
-            if sub_item["kind"] == "PARM_DECL":
-                parameter_type_list.append(f'"{sub_item["name"]}"_a')
+            if sub_item["cursor_kind"]["name"] == "PARM_DECL":
+                parameter_type_list.append(f'"{sub_item["cursor"]["spelling"]}"_a')
 
         parameter_type_list = ",".join(parameter_type_list)
         if parameter_type_list:
@@ -317,7 +325,7 @@ def handle_constructor(self) -> None:
 
         # generate parameter type list
         for sub_item in self.members:
-            if sub_item["kind"] == "PARM_DECL":
+            if sub_item["cursor_kind"]["name"] == "PARM_DECL":
                 parameter_type_list.append(self.get_parm_types(sub_item))
         parameter_type_list = ",".join(parameter_type_list)
 
@@ -326,25 +334,29 @@ def handle_constructor(self) -> None:
             self._linelist.append(f".def(py::init<{parameter_type_list}>())")
 
     def get_parm_types(self, item: Dict[str, Any]) -> List[str]:
-        if item["element_type"] == "LValueReference":
+        if item["type"]["kind"] == "LValueReference":
             for sub_item in item["members"]:
-                if sub_item["kind"] == "TYPE_REF":
+                if sub_item["cursor_kind"]["name"] == "TYPE_REF":
                     # @TODO: Make more robust
                     type_ref = (
-                        sub_item["name"].replace("struct ", "").replace("pcl::", "")
+                        sub_item["cursor"]["spelling"]
+                        .replace("struct ", "")
+                        .replace("pcl::", "")
                     )
                     parameter_type_list = f"{type_ref} &"
-        elif item["element_type"] == "Elaborated":
+        elif item["type"]["kind"] == "Elaborated":
             namespace_ref = ""
             for sub_item in item["members"]:
-                if sub_item["kind"] == "NAMESPACE_REF":
-                    namespace_ref += f'{sub_item["name"]}::'
-                if sub_item["kind"] == "TYPE_REF":
-                    parameter_type_list = f'{namespace_ref}{sub_item["name"]}'
-        elif item["element_type"] in ("Float", "Double", "Int"):
-            parameter_type_list = f'{item["element_type"].lower()}'
+                if sub_item["cursor_kind"]["name"] == "NAMESPACE_REF":
+                    namespace_ref += f'{sub_item["cursor"]["spelling"]}::'
+                if sub_item["cursor_kind"]["name"] == "TYPE_REF":
+                    parameter_type_list = (
+                        f'{namespace_ref}{sub_item["cursor"]["spelling"]}'
+                    )
+        elif item["type"]["kind"] in ("Float", "Double", "Int"):
+            parameter_type_list = f'{item["type"]["kind"].lower()}'
         else:
-            parameter_type_list = f'{item["element_type"]}'
+            parameter_type_list = f'{item["type"]["kind"]}'
         return parameter_type_list
 
     def handle_inclusion_directive(self) -> None:
@@ -417,7 +429,7 @@ def combine_lines() -> list or Exception:
     if parsed_info:
         bind_object = bind(root=parsed_info, module_name=module_name)
         # Extract filename from parsed_info (TRANSLATION_UNIT's name contains the filepath)
-        filename = "pcl" + parsed_info["name"].rsplit("pcl")[-1]
+        filename = "pcl" + parsed_info["cursor"]["spelling"].rsplit("pcl")[-1]
         return combine_lines()
     else:
         raise Exception("Empty dict: parsed_info")