Add Graph Execution-Order Diagnostics to All-Ranks Landing Page (#145)

**PR Summary**
This change integrates the execution-order report (built in
#140) into the --all-ranks-html
landing page, surfacing a new “Graph Execution-Order Diagnostics”
section. The page now highlights whether graph execution order differs
across ranks and summarizes which ranks show collective schedule
mismatches or cache hit/miss divergences, reusing existing divergence
checks without introducing any CLI changes.

<img width="1708" height="676" alt="Screenshot 2025-08-27 at 10 04
37 PM"
src="https://github.com/user-attachments/assets/a057c49c-178e-45bc-a251-f4eaba78e214"
/>

Author: skarjala

src/cli.rs

@@ -6,7 +6,7 @@ use std::path::PathBuf;
 
 use fxhash::{FxHashMap, FxHashSet};
 use tlparse::{
-    analyze_graph_runtime_deltas, generate_multi_rank_html, parse_path,
+    analyze_graph_runtime_deltas, build_exec_order_summary, generate_multi_rank_html, parse_path,
     read_chromium_events_with_pid, ArtifactFlags, Diagnostics, DivergenceFlags, DivergenceGroup,
     ParseConfig, RankMetaData,
 };
@@ -549,6 +549,8 @@ fn handle_all_ranks(
         out_path.display()
     );
 
+    let exec_order_summary = build_exec_order_summary(&out_path, &rank_nums, &collective_schedules);
+
     let diagnostics = Diagnostics {
         divergence: DivergenceFlags {
             cache: cache_seq_groups.len() > 1,
@@ -562,6 +564,7 @@ fn handle_all_ranks(
         cache_groups: cache_divergence_groups.clone(),
         collective_groups: collective_divergence_groups.clone(),
         tensor_meta_groups: tensor_meta_divergence_groups.clone(),
+        exec_order: exec_order_summary,
     };
 
     let (landing_page_path, landing_html) = generate_multi_rank_html(

src/lib.rs

@@ -25,9 +25,9 @@ mod templates;
 mod types;
 
 pub use types::{
-    ArtifactFlags, CollectivesParityReport, Diagnostics, DivergenceFlags, DivergenceGroup,
-    GraphAnalysis, GraphCollectivesParity, GraphRuntime, RankMetaData, RuntimeAnalysis,
-    RuntimeRankDetail,
+    ArtifactFlags, CollectiveSchedule, CollectivesParityReport, Diagnostics, DivergenceFlags,
+    DivergenceGroup, ExecOrderSummary, GraphAnalysis, GraphCollectivesParity, GraphRuntime,
+    RankMetaData, RuntimeAnalysis, RuntimeRankDetail,
 };
 
 pub use execution_order::{
@@ -1290,6 +1290,220 @@ pub fn generate_multi_rank_html(
     Ok((landing_page_path, html))
 }
 
+/// Build ExecOrderSummary from artifacts under out_path for the given ranks
+pub fn build_exec_order_summary(
+    out_path: &PathBuf,
+    rank_nums: &[u32],
+    collective_schedules: &[CollectiveSchedule],
+) -> Option<ExecOrderSummary> {
+    use crate::execution_order::{
+        analyze_execution_order, parse_graph_execution_order, ExecOrderIssue,
+    };
+    use std::collections::HashSet;
+
+    // Preload and parse compile_directory.json per rank
+    let cd_by_rank: FxHashMap<u32, serde_json::Map<String, serde_json::Value>> = rank_nums
+        .iter()
+        .filter_map(|&rank| {
+            let path = out_path
+                .join(format!("rank_{rank}"))
+                .join("compile_directory.json");
+            std::fs::read_to_string(path)
+                .ok()
+                .and_then(|content| serde_json::from_str::<serde_json::Value>(&content).ok())
+                .and_then(|val| match val {
+                    serde_json::Value::Object(map) => Some((rank, map)),
+                    _ => None,
+                })
+        })
+        .collect();
+
+    // Collect latest "graph_execution" artifact per rank
+    let exec_orders: FxHashMap<u32, Vec<String>> = rank_nums
+        .iter()
+        .filter_map(|&rank| {
+            let map = cd_by_rank.get(&rank)?;
+            let rank_dir = out_path.join(format!("rank_{rank}"));
+
+            // Find latest graph_execution artifact
+            let best = map
+                .values()
+                .filter_map(|entry| entry.get("artifacts")?.as_array())
+                .flat_map(|arts| arts.iter())
+                .filter_map(|a| {
+                    let name = a.get("name")?.as_str()?;
+                    if !name.contains("graph_execution") || !name.ends_with(".json") {
+                        return None;
+                    }
+                    Some((
+                        a.get("number")?.as_u64()?,
+                        a.get("url")?.as_str()?.to_string(),
+                    ))
+                })
+                .max_by_key(|(num, _)| *num)?;
+
+            let path = rank_dir.join(best.1);
+            let order = std::fs::read_to_string(path)
+                .ok()
+                .and_then(|payload| parse_graph_execution_order(&payload).ok())
+                .map(|order| order.into_iter().map(|s| format!("[{}]", s)).collect())?;
+            Some((rank, order))
+        })
+        .collect();
+
+    if exec_orders.len() < 2 {
+        return None;
+    }
+
+    // Build dir -> compile_id mapping per rank
+    let dir_to_compile_id_per_rank: FxHashMap<u32, FxHashMap<String, String>> = rank_nums
+        .iter()
+        .filter_map(|&rank| {
+            let obj = cd_by_rank.get(&rank)?;
+            let mapping = obj
+                .iter()
+                .flat_map(|(cid, entry)| {
+                    entry
+                        .get("artifacts")
+                        .and_then(|x| x.as_array())
+                        .map(|arts| {
+                            arts.iter()
+                                .filter_map(|a| {
+                                    let url = a.get("url")?.as_str()?;
+                                    let prefix = url.split_once('/')?.0;
+                                    Some((prefix.to_string(), cid.to_string()))
+                                })
+                                .collect::<Vec<_>>()
+                        })
+                        .unwrap_or_default()
+                })
+                .fold(FxHashMap::default(), |mut acc, (prefix, cid)| {
+                    acc.entry(prefix).or_insert(cid);
+                    acc
+                });
+            Some((rank, mapping))
+        })
+        .collect();
+
+    // Build collective ops mapping
+    let collective_by_graph: FxHashMap<(u32, String), Vec<String>> = collective_schedules
+        .iter()
+        .filter_map(|cs| {
+            let m = dir_to_compile_id_per_rank.get(&cs.rank)?;
+            let compile_id = m
+                .get(&cs.graph)
+                .cloned()
+                .unwrap_or_else(|| cs.graph.clone());
+            Some(((cs.rank, compile_id), cs.ops.clone()))
+        })
+        .fold(FxHashMap::default(), |mut acc, (key, ops)| {
+            acc.entry(key).or_default().extend(ops);
+            acc
+        });
+
+    // Build cache status mapping
+    let cache_status: FxHashMap<(u32, String), String> = rank_nums
+        .iter()
+        .flat_map(|&rank| {
+            cd_by_rank
+                .get(&rank)
+                .and_then(|obj| {
+                    dir_to_compile_id_per_rank
+                        .get(&rank)
+                        .map(|dir2cid| (obj, dir2cid))
+                })
+                .map(|(obj, dir2cid)| {
+                    let status_by_dir = obj
+                        .values()
+                        .filter_map(|entry| entry.get("artifacts").and_then(|x| x.as_array()))
+                        .flat_map(|arts| arts.iter())
+                        .filter_map(|a| {
+                            let url = a.get("url")?.as_str()?;
+                            let prefix = url.split_once('/')?.0.to_string();
+                            let name = a.get("name")?.as_str()?;
+                            let status = match () {
+                                _ if name.contains("cache_miss") => ("miss", 3),
+                                _ if name.contains("cache_hit") => ("hit", 2),
+                                _ if name.contains("cache_bypass") => ("bypass", 1),
+                                _ => return None,
+                            };
+                            Some((prefix, status))
+                        })
+                        .fold(
+                            FxHashMap::default(),
+                            |mut acc, (prefix, (status, priority))| {
+                                acc.entry(prefix)
+                                    .and_modify(|e: &mut (&str, u8)| {
+                                        if priority > e.1 {
+                                            *e = (status, priority);
+                                        }
+                                    })
+                                    .or_insert((status, priority));
+                                acc
+                            },
+                        );
+
+                    status_by_dir
+                        .into_iter()
+                        .filter_map(|(dir, (st, _))| {
+                            let cid = dir2cid.get(&dir)?;
+                            Some(((rank, cid.clone()), st.to_string()))
+                        })
+                        .collect::<Vec<_>>()
+                })
+                .unwrap_or_default()
+        })
+        .collect();
+
+    // Analyze and create summary
+    let report = analyze_execution_order(&exec_orders, &collective_by_graph, &cache_status);
+
+    let order_differs = report.by_index.iter().any(|row| {
+        let uniq: HashSet<_> = row.by_rank.values().map(String::as_str).collect();
+        uniq.len() > 1
+    });
+
+    let (sched_set, cache_set) = report.by_index.iter().fold(
+        (HashSet::new(), HashSet::new()),
+        |(mut sched, mut cache), row| {
+            if row.issues.contains(&ExecOrderIssue::ScheduleMismatch) {
+                sched.extend(row.by_rank.keys());
+            }
+            if row.issues.contains(&ExecOrderIssue::CacheMismatch) {
+                cache.extend(row.by_rank.keys());
+            }
+            (sched, cache)
+        },
+    );
+
+    let mut ranks_schedule: Vec<u32> = sched_set.into_iter().collect();
+    let mut ranks_cache: Vec<u32> = cache_set.into_iter().collect();
+    ranks_schedule.sort_unstable();
+    ranks_cache.sort_unstable();
+
+    let format_ranks = |ranks: &[u32]| {
+        if ranks.is_empty() {
+            String::new()
+        } else {
+            ranks
+                .iter()
+                .map(|r| format!("Rank {r}"))
+                .collect::<Vec<_>>()
+                .join(", ")
+        }
+    };
+
+    Some(ExecOrderSummary {
+        order_differs,
+        has_schedule_mismatch: !ranks_schedule.is_empty(),
+        has_cache_mismatch: !ranks_cache.is_empty(),
+        ranks_schedule_str: format_ranks(&ranks_schedule),
+        ranks_cache_str: format_ranks(&ranks_cache),
+        ranks_schedule,
+        ranks_cache,
+    })
+}
+
 fn prepare_and_validate_graphs(
     runtime_estimations: &[GraphRuntime],
 ) -> Option<(
@@ -1863,7 +2077,7 @@ pub mod execution_order {
             // Evaluate issues among present ranks
             let mut issues: Vec<ExecOrderIssue> = Vec::new();
 
-            // Schedule mismatch: compare collective op sequences
+            // Schedule mismatch: compare collective schedules regardless of compile_id
             {
                 let schedules: Vec<Vec<String>> = by_rank
                     .iter()
@@ -1880,7 +2094,7 @@ pub mod execution_order {
                 }
             }
 
-            // Cache skew: compare cache status markers
+            // Cache mismatch: compare cache statuses regardless of compile_id
             {
                 let statuses: Vec<String> = by_rank
                     .iter()

src/templates.rs

@@ -621,6 +621,27 @@ desync issues on specific ranks.
 {{ endfor }}
 {{ endif }}
 {{ endif }}
+<h3>Graph Execution-Order Diagnostics</h3>
+<p>Note: To enable this feature, wrap your code with torch._inductor.debug.record_and_log_graph_execution_order()</p>
+{{ if diagnostics.exec_order }}
+  {{ if diagnostics.exec_order.order_differs }}
+  <p>Graph execution order differs across ranks.</p>
+  {{ else }}
+  <p>Graph execution order: consistent across ranks.</p>
+  {{ endif }}
+  {{ if diagnostics.exec_order.has_schedule_mismatch }}
+  <p><strong>Warning:</strong> Schedule mismatch across ranks: {diagnostics.exec_order.ranks_schedule_str}</p>
+  {{ else }}
+  <p>Collectives Schedule: Consistent across ranks.</p>
+  {{ endif }}
+  {{ if diagnostics.exec_order.has_cache_mismatch }}
+  <p><strong>Warning:</strong> Cache hit/miss mismatch across ranks: {diagnostics.exec_order.ranks_cache_str}</p>
+  {{ else }}
+  <p>Cache hit/miss sequence: Consistent across ranks.</p>
+  {{ endif }}
+{{ else }}
+<p>Execution-order analysis unavailable.</p>
+{{ endif }}
 <h3>Tensor Metadata Analysis</h3>
 <p>
 Compares inductor tensor metadata (shapes, dtypes, strides) across ranks to detect compilation divergence.

src/types.rs

@@ -929,6 +929,24 @@ pub struct ArtifactFlags {
     pub runtime_trace: bool,
 }
 
+#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
+pub struct ExecOrderSummary {
+    /// True if any index has differing compile_ids across ranks
+    pub order_differs: bool,
+    /// Ranks involved in any schedule mismatches (sorted numerically)
+    pub ranks_schedule: Vec<u32>,
+    /// Ranks involved in any cache hit/miss mismatches (sorted numerically)
+    pub ranks_cache: Vec<u32>,
+    /// True if there is any schedule mismatch
+    pub has_schedule_mismatch: bool,
+    /// True if there is any cache hit/miss mismatch
+    pub has_cache_mismatch: bool,
+    /// Pretty-printed ranks for template rendering (e.g., "r0, r1, r2")
+    pub ranks_schedule_str: String,
+    /// Pretty-printed ranks for template rendering (e.g., "r0, r2")
+    pub ranks_cache_str: String,
+}
+
 #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
 pub struct Diagnostics {
     pub divergence: DivergenceFlags,
@@ -937,6 +955,7 @@ pub struct Diagnostics {
     pub cache_groups: Vec<DivergenceGroup>,
     pub collective_groups: Vec<DivergenceGroup>,
     pub tensor_meta_groups: Vec<DivergenceGroup>,
+    pub exec_order: Option<ExecOrderSummary>,
 }
 
 #[derive(Serialize)]