Add basic support for augmentsSyntaxTokens

Author: Veykril

crates/ide/src/syntax_highlighting.rs

@@ -469,25 +469,8 @@ fn traverse(
             }
 
             // apply config filtering
-            match &mut highlight.tag {
-                HlTag::StringLiteral if !config.strings => continue,
-                // If punctuation is disabled, make the macro bang part of the macro call again.
-                tag @ HlTag::Punctuation(HlPunct::MacroBang) => {
-                    if !config.macro_bang {
-                        *tag = HlTag::Symbol(SymbolKind::Macro);
-                    } else if !config.specialize_punctuation {
-                        *tag = HlTag::Punctuation(HlPunct::Other);
-                    }
-                }
-                HlTag::Punctuation(_) if !config.punctuation => continue,
-                tag @ HlTag::Punctuation(_) if !config.specialize_punctuation => {
-                    *tag = HlTag::Punctuation(HlPunct::Other);
-                }
-                HlTag::Operator(_) if !config.operator && highlight.mods.is_empty() => continue,
-                tag @ HlTag::Operator(_) if !config.specialize_operator => {
-                    *tag = HlTag::Operator(HlOperator::Other);
-                }
-                _ => (),
+            if !filter_by_config(&mut highlight, config) {
+                continue;
             }
 
             if inside_attribute {
@@ -498,3 +481,27 @@ fn traverse(
         }
     }
 }
+
+fn filter_by_config(highlight: &mut Highlight, config: HighlightConfig) -> bool {
+    match &mut highlight.tag {
+        HlTag::StringLiteral if !config.strings => return false,
+        // If punctuation is disabled, make the macro bang part of the macro call again.
+        tag @ HlTag::Punctuation(HlPunct::MacroBang) => {
+            if !config.macro_bang {
+                *tag = HlTag::Symbol(SymbolKind::Macro);
+            } else if !config.specialize_punctuation {
+                *tag = HlTag::Punctuation(HlPunct::Other);
+            }
+        }
+        HlTag::Punctuation(_) if !config.punctuation => return false,
+        tag @ HlTag::Punctuation(_) if !config.specialize_punctuation => {
+            *tag = HlTag::Punctuation(HlPunct::Other);
+        }
+        HlTag::Operator(_) if !config.operator && highlight.mods.is_empty() => return false,
+        tag @ HlTag::Operator(_) if !config.specialize_operator => {
+            *tag = HlTag::Operator(HlOperator::Other);
+        }
+        _ => (),
+    }
+    true
+}

crates/rust-analyzer/src/config.rs

@@ -1028,6 +1028,11 @@ impl Config {
         .is_some()
     }
 
+    pub fn semantics_tokens_augments_syntax_tokens(&self) -> bool {
+        try_!(self.caps.text_document.as_ref()?.semantic_tokens.as_ref()?.augments_syntax_tokens?)
+            .unwrap_or(false)
+    }
+
     pub fn position_encoding(&self) -> PositionEncoding {
         negotiated_encoding(&self.caps)
     }

crates/rust-analyzer/src/handlers/request.rs

@@ -1472,7 +1472,12 @@ pub(crate) fn handle_semantic_tokens_full(
         snap.workspaces.is_empty() || !snap.proc_macros_loaded;
 
     let highlights = snap.analysis.highlight(highlight_config, file_id)?;
-    let semantic_tokens = to_proto::semantic_tokens(&text, &line_index, highlights);
+    let semantic_tokens = to_proto::semantic_tokens(
+        &text,
+        &line_index,
+        highlights,
+        snap.config.semantics_tokens_augments_syntax_tokens(),
+    );
 
     // Unconditionally cache the tokens
     snap.semantic_tokens_cache.lock().insert(params.text_document.uri, semantic_tokens.clone());
@@ -1496,7 +1501,12 @@ pub(crate) fn handle_semantic_tokens_full_delta(
         snap.workspaces.is_empty() || !snap.proc_macros_loaded;
 
     let highlights = snap.analysis.highlight(highlight_config, file_id)?;
-    let semantic_tokens = to_proto::semantic_tokens(&text, &line_index, highlights);
+    let semantic_tokens = to_proto::semantic_tokens(
+        &text,
+        &line_index,
+        highlights,
+        snap.config.semantics_tokens_augments_syntax_tokens(),
+    );
 
     let mut cache = snap.semantic_tokens_cache.lock();
     let cached_tokens = cache.entry(params.text_document.uri).or_default();
@@ -1530,7 +1540,12 @@ pub(crate) fn handle_semantic_tokens_range(
         snap.workspaces.is_empty() || !snap.proc_macros_loaded;
 
     let highlights = snap.analysis.highlight_range(highlight_config, frange)?;
-    let semantic_tokens = to_proto::semantic_tokens(&text, &line_index, highlights);
+    let semantic_tokens = to_proto::semantic_tokens(
+        &text,
+        &line_index,
+        highlights,
+        snap.config.semantics_tokens_augments_syntax_tokens(),
+    );
     Ok(Some(semantic_tokens.into()))
 }
 

crates/rust-analyzer/src/to_proto.rs

@@ -586,6 +586,7 @@ pub(crate) fn semantic_tokens(
     text: &str,
     line_index: &LineIndex,
     highlights: Vec<HlRange>,
+    semantics_tokens_augments_syntax_tokens: bool,
 ) -> lsp_types::SemanticTokens {
     let id = TOKEN_RESULT_COUNTER.fetch_add(1, Ordering::SeqCst).to_string();
     let mut builder = semantic_tokens::SemanticTokensBuilder::new(id);
@@ -595,6 +596,26 @@ pub(crate) fn semantic_tokens(
             continue;
         }
 
+        if semantics_tokens_augments_syntax_tokens {
+            match highlight_range.highlight.tag {
+                HlTag::BoolLiteral
+                | HlTag::ByteLiteral
+                | HlTag::CharLiteral
+                | HlTag::Comment
+                | HlTag::Keyword
+                | HlTag::NumericLiteral
+                | HlTag::Operator(_)
+                | HlTag::Punctuation(_)
+                | HlTag::StringLiteral
+                | HlTag::None
+                    if highlight_range.highlight.mods.is_empty() =>
+                {
+                    continue
+                }
+                _ => (),
+            }
+        }
+
         let (ty, mods) = semantic_token_type_and_modifiers(highlight_range.highlight);
         let token_index = semantic_tokens::type_index(ty);
         let modifier_bitset = mods.0;