Workaround for expansion of function-like macros

This commit resolves an issue where macros that evaluate to a constant
but have a function like macro in the macro body would not be properly
expanded by cexpr.

This adds an opt-in option to use Clang on intermediary files to
evaluate the macros one by one. This is opt-in largely because of the
compile time implications.

Author: jbaublitz

bindgen-cli/options.rs

@@ -400,6 +400,12 @@ struct BindgenCommand {
     /// Wrap unsafe operations in unsafe blocks.
     #[arg(long)]
     wrap_unsafe_ops: bool,
+    /// Enable fallback for clang macro parsing.
+    #[arg(long)]
+    clang_macro_fallback: bool,
+    /// Set path for temporary files generated by fallback for clang macro parsing.
+    #[arg(long)]
+    clang_macro_fallback_build_dir: Option<PathBuf>,
     /// Derive custom traits on any kind of type. The CUSTOM value must be of the shape REGEX=DERIVE where DERIVE is a coma-separated list of derive macros.
     #[arg(long, value_name = "CUSTOM", value_parser = parse_custom_derive)]
     with_derive_custom: Vec<(Vec<String>, String)>,
@@ -554,6 +560,8 @@ where
         merge_extern_blocks,
         override_abi,
         wrap_unsafe_ops,
+        clang_macro_fallback,
+        clang_macro_fallback_build_dir,
         with_derive_custom,
         with_derive_custom_struct,
         with_derive_custom_enum,
@@ -1023,6 +1031,14 @@ where
         builder = builder.wrap_unsafe_ops(true);
     }
 
+    if clang_macro_fallback {
+        builder = builder.clang_macro_fallback();
+    }
+
+    if let Some(path) = clang_macro_fallback_build_dir {
+        builder = builder.clang_macro_fallback_build_dir(path);
+    }
+
     #[derive(Debug)]
     struct CustomDeriveCallback {
         derives: Vec<String>,

bindgen-tests/tests/expectations/tests/issue-753.rs

@@ -0,0 +1,7 @@
+// bindgen-flags: --clang-macro-fallback
+
+#define UINT32_C(c) c ## U
+
+#define CONST UINT32_C(5)
+#define OTHER_CONST UINT32_C(6)
+#define LARGE_CONST UINT32_C(6 << 8)

bindgen-tests/tests/expectations/tests/libclang-9/issue-753.rs

@@ -10,6 +10,7 @@ use std::cmp;
 
 use std::ffi::{CStr, CString};
 use std::fmt;
+use std::fs::OpenOptions;
 use std::hash::Hash;
 use std::hash::Hasher;
 use std::os::raw::{c_char, c_int, c_longlong, c_uint, c_ulong, c_ulonglong};
@@ -1868,6 +1869,27 @@ impl TranslationUnit {
         }
     }
 
+    /// Save a translation unit to the given file.
+    pub(crate) fn save(&mut self, file: &str) -> Result<(), CXSaveError> {
+        let file = if let Ok(cstring) = CString::new(file) {
+            cstring
+        } else {
+            return Err(CXSaveError_Unknown);
+        };
+        let ret = unsafe {
+            clang_saveTranslationUnit(
+                self.x,
+                file.as_ptr(),
+                clang_defaultSaveOptions(self.x),
+            )
+        };
+        if ret != 0 {
+            Err(ret)
+        } else {
+            Ok(())
+        }
+    }
+
     /// Is this the null translation unit?
     pub(crate) fn is_null(&self) -> bool {
         self.x.is_null()
@@ -1882,6 +1904,89 @@ impl Drop for TranslationUnit {
     }
 }
 
+/// Translation unit used for macro fallback parsing
+pub(crate) struct FallbackTranslationUnit {
+    file_path: String,
+    pch_paths: Vec<String>,
+    idx: Box<Index>,
+    tu: TranslationUnit,
+}
+
+impl fmt::Debug for FallbackTranslationUnit {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(fmt, "FallbackTranslationUnit {{ }}")
+    }
+}
+
+impl FallbackTranslationUnit {
+    /// Create a new fallback translation unit
+    pub(crate) fn new(
+        file: String,
+        pch_paths: Vec<String>,
+        c_args: &[Box<str>],
+    ) -> Option<Self> {
+        // Create empty file
+        OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(&file)
+            .ok()?;
+
+        let f_index = Box::new(Index::new(true, false));
+        let f_translation_unit = TranslationUnit::parse(
+            &f_index,
+            &file,
+            c_args,
+            &[],
+            CXTranslationUnit_None,
+        )?;
+        Some(FallbackTranslationUnit {
+            file_path: file,
+            pch_paths,
+            tu: f_translation_unit,
+            idx: f_index,
+        })
+    }
+
+    /// Get reference to underlying translation unit.
+    pub(crate) fn translation_unit(&self) -> &TranslationUnit {
+        &self.tu
+    }
+
+    /// Reparse a translation unit.
+    pub(crate) fn reparse(
+        &mut self,
+        unsaved_contents: &str,
+    ) -> Result<(), CXErrorCode> {
+        let unsaved = &[UnsavedFile::new(&self.file_path, unsaved_contents)];
+        let mut c_unsaved: Vec<CXUnsavedFile> =
+            unsaved.iter().map(|f| f.x).collect();
+        let ret = unsafe {
+            clang_reparseTranslationUnit(
+                self.tu.x,
+                unsaved.len() as c_uint,
+                c_unsaved.as_mut_ptr(),
+                clang_defaultReparseOptions(self.tu.x),
+            )
+        };
+        if ret != 0 {
+            Err(ret)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl Drop for FallbackTranslationUnit {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_file(&self.file_path);
+        for pch in self.pch_paths.iter() {
+            let _ = std::fs::remove_file(pch);
+        }
+    }
+}
+
 /// A diagnostic message generated while parsing a translation unit.
 pub(crate) struct Diagnostic {
     x: CXDiagnostic,

bindgen-tests/tests/headers/issue-753.h

@@ -30,6 +30,7 @@ use std::borrow::Cow;
 use std::cell::{Cell, RefCell};
 use std::collections::{BTreeSet, HashMap as StdHashMap};
 use std::mem;
+use std::path::Path;
 
 /// An identifier for some kind of IR item.
 #[derive(Debug, Copy, Clone, Eq, PartialOrd, Ord, Hash)]
@@ -376,6 +377,9 @@ pub(crate) struct BindgenContext {
     /// The translation unit for parsing.
     translation_unit: clang::TranslationUnit,
 
+    /// The translation unit for macro fallback parsing.
+    fallback_tu: Option<clang::FallbackTranslationUnit>,
+
     /// Target information that can be useful for some stuff.
     target_info: clang::TargetInfo,
 
@@ -584,6 +588,7 @@ If you encounter an error missing from this list, please file an issue or a PR!"
             collected_typerefs: false,
             in_codegen: false,
             translation_unit,
+            fallback_tu: None,
             target_info,
             options,
             generated_bindgen_complex: Cell::new(false),
@@ -2060,6 +2065,77 @@ If you encounter an error missing from this list, please file an issue or a PR!"
         &self.translation_unit
     }
 
+    /// Initialize fallback translation unit if it does not exist and
+    /// then return a mutable reference to the fallback translation unit.
+    pub(crate) fn try_ensure_fallback_translation_unit(
+        &mut self,
+    ) -> Option<&mut clang::FallbackTranslationUnit> {
+        if self.fallback_tu.is_none() {
+            let file = format!(
+                "{}/.macro_eval.c",
+                match self.options().clang_macro_fallback_build_dir {
+                    Some(ref path) => path.as_os_str().to_str()?,
+                    None => ".",
+                }
+            );
+
+            let index = clang::Index::new(false, false);
+
+            let mut c_args = Vec::new();
+            let mut pch_paths = Vec::new();
+            for input_header in self.options().input_headers.iter() {
+                let path = Path::new(input_header.as_ref());
+                let header_name = path
+                    .file_name()
+                    .and_then(|hn| hn.to_str())
+                    .map(|s| s.to_owned());
+                let header_path = path
+                    .parent()
+                    .and_then(|hp| hp.to_str())
+                    .map(|s| s.to_owned());
+
+                let (header, pch) = if let (Some(ref hp), Some(hn)) =
+                    (header_path, header_name)
+                {
+                    let header_path = if hp.is_empty() { "." } else { hp };
+                    let header = format!("{header_path}/{hn}");
+                    let pch_path = if let Some(ref path) =
+                        self.options().clang_macro_fallback_build_dir
+                    {
+                        path.as_os_str().to_str()?
+                    } else {
+                        header_path
+                    };
+                    (header, format!("{pch_path}/{hn}.pch"))
+                } else {
+                    return None;
+                };
+
+                let mut tu = clang::TranslationUnit::parse(
+                    &index,
+                    &header,
+                    &[
+                        "-x".to_owned().into_boxed_str(),
+                        "c-header".to_owned().into_boxed_str(),
+                    ],
+                    &[],
+                    clang_sys::CXTranslationUnit_ForSerialization,
+                )?;
+                tu.save(&pch).ok()?;
+
+                c_args.push("-include-pch".to_string().into_boxed_str());
+                c_args.push(pch.clone().into_boxed_str());
+                pch_paths.push(pch);
+            }
+
+            self.fallback_tu = Some(clang::FallbackTranslationUnit::new(
+                file, pch_paths, &c_args,
+            )?);
+        }
+
+        self.fallback_tu.as_mut()
+    }
+
     /// Have we parsed the macro named `macro_name` already?
     pub(crate) fn parsed_macro(&self, macro_name: &[u8]) -> bool {
         self.parsed_macros.contains_key(macro_name)

bindgen/clang.rs

@@ -389,9 +389,51 @@ impl ClangSubItemParser for Var {
     }
 }
 
+/// This function uses a [`FallbackTranslationUnit`][clang::FallbackTranslationUnit] to parse each
+/// macro that cannot be parsed by the normal bindgen process for `#define`s.
+///
+/// To construct the [`FallbackTranslationUnit`][clang::FallbackTranslationUnit], first precompiled
+/// headers are generated for all input headers. An empty temporary `.c` file is generated to pass
+/// to the translation unit. On the evaluation of each macro, a [`String`] is generated with the
+/// new contents of the empty file and passed in for reparsing. The precompiled headers and
+/// preservation of the [`FallbackTranslationUnit`][clang::FallbackTranslationUnit] across macro
+/// evaluations are both optimizations that have significantly improved the performance.
+fn parse_macro_clang_fallback(
+    ctx: &mut BindgenContext,
+    cursor: &clang::Cursor,
+) -> Option<(Vec<u8>, cexpr::expr::EvalResult)> {
+    if !ctx.options().clang_macro_fallback {
+        return None;
+    }
+
+    let ftu = ctx.try_ensure_fallback_translation_unit()?;
+    let contents = format!("int main() {{ {}; }}", cursor.spelling(),);
+    ftu.reparse(&contents).ok()?;
+    // Children of root node of AST
+    let root_children = ftu.translation_unit().cursor().collect_children();
+    // Last child in root is function declaration
+    // Should be FunctionDecl
+    let main_func = root_children.last()?;
+    // Children should all be statements in function declaration
+    let all_stmts = main_func.collect_children();
+    // First child in all_stmts should be the statement containing the macro to evaluate
+    // Should be CompoundStmt
+    let macro_stmt = all_stmts.first()?;
+    // Children should all be expressions from the compound statement
+    let paren_exprs = macro_stmt.collect_children();
+    // First child in all_exprs is the expression utilizing the given macro to be evaluated
+    // Should  be ParenExpr
+    let paren = paren_exprs.first()?;
+
+    Some((
+        cursor.spelling().into_bytes(),
+        cexpr::expr::EvalResult::Int(Wrapping(paren.evaluate()?.as_int()?)),
+    ))
+}
+
 /// Try and parse a macro using all the macros parsed until now.
 fn parse_macro(
-    ctx: &BindgenContext,
+    ctx: &mut BindgenContext,
     cursor: &clang::Cursor,
 ) -> Option<(Vec<u8>, cexpr::expr::EvalResult)> {
     use cexpr::expr;
@@ -402,7 +444,7 @@ fn parse_macro(
 
     match parser.macro_definition(&cexpr_tokens) {
         Ok((_, (id, val))) => Some((id.into(), val)),
-        _ => None,
+        _ => parse_macro_clang_fallback(ctx, cursor),
     }
 }
 

bindgen/ir/context.rs

@@ -21,9 +21,7 @@ use crate::HashMap;
 use crate::DEFAULT_ANON_FIELDS_PREFIX;
 
 use std::env;
-#[cfg(feature = "experimental")]
-use std::path::Path;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::rc::Rc;
 
 use as_args::AsArgs;
@@ -2107,5 +2105,38 @@ options! {
             }
         },
         as_args: "--emit-diagnostics",
+    },
+    /// Whether to use Clang evaluation on temporary files as a fallback for macros that fail to
+    /// parse.
+    clang_macro_fallback: bool {
+        methods: {
+            /// Use Clang as a fallback for macros that fail to parse using `CExpr`.
+            ///
+            /// This uses a workaround to evaluate each macro in a temporary file. Because this
+            /// results in slower compilation, this option is opt-in.
+            pub fn clang_macro_fallback(mut self) -> Self {
+                self.options.clang_macro_fallback = true;
+                self
+            }
+        },
+        as_args: "--clang-macro-fallback",
+    }
+    /// Path to use for temporary files created by clang macro fallback code like precompiled
+    /// headers.
+    clang_macro_fallback_build_dir: Option<PathBuf> {
+        methods: {
+            /// Set a path to a directory to which `.c` and `.h.pch` files should be written for the
+            /// purpose of using clang to evaluate macros that can't be easily parsed.
+            ///
+            /// The default location for `.h.pch` files is the directory that the corresponding
+            /// `.h` file is located in. The default for the temporary `.c` file used for clang
+            /// parsing is the current working directory. Both of these defaults are overridden
+            /// by this option.
+            pub fn clang_macro_fallback_build_dir<P: AsRef<Path>>(mut self, path: P) -> Self {
+                self.options.clang_macro_fallback_build_dir = Some(path.as_ref().to_owned());
+                self
+            }
+        },
+        as_args: "--clang-macro-fallback-build-dir",
     }
 }