Topic properties auto

__tests__/tools/topic_property_extractor.test.js

@@ -0,0 +1,58 @@
+const path = require('path');
+const fs = require('fs');
+const { execSync } = require('child_process');
+
+describe('topic_property_extractor.py', () => {
+  const scriptPath = path.resolve(__dirname, '../../tools/property-extractor/topic_property_extractor.py');
+  const mockSourcePath = path.resolve(__dirname, 'mock-redpanda-src');
+  const outputJson = path.resolve(__dirname, 'topic-properties-output.json');
+  const outputAdoc = path.resolve(__dirname, 'topic-properties.adoc');
+
+  beforeAll(() => {
+    // Create a minimal mock Redpanda source tree
+    if (!fs.existsSync(mockSourcePath)) {
+      fs.mkdirSync(mockSourcePath, { recursive: true });
+      // Create a mock header file with a topic property
+      const headerDir = path.join(mockSourcePath, 'src/v/kafka/server/handlers/topics');
+      fs.mkdirSync(headerDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(headerDir, 'types.h'),
+        'inline constexpr std::string_view topic_property_retention_ms = "retention.ms";\n'
+      );
+      // Add a mock .cc file (should be ignored for property extraction)
+      fs.writeFileSync(
+        path.join(headerDir, 'types.cc'),
+        `// Copyright 2025 Redpanda Data, Inc.\n#include "kafka/server/handlers/topics/types.h"\n// ...rest of the file...\n`
+      );
+      // Add a mock config file to simulate a cluster property mapping
+      const configDir = path.join(mockSourcePath, 'src/v/config');
+      fs.mkdirSync(configDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(configDir, 'mock_config.cc'),
+        'config.get("log_retention_ms");\n'
+      );
+    }
+  });
+
+  afterAll(() => {
+    // Cleanup
+    if (fs.existsSync(outputJson)) fs.unlinkSync(outputJson);
+    if (fs.existsSync(outputAdoc)) fs.unlinkSync(outputAdoc);
+    fs.rmdirSync(mockSourcePath, { recursive: true });
+  });
+
+  it('extracts topic properties and generates JSON', () => {
+    execSync(`python3 ${scriptPath} --source-path ${mockSourcePath} --output-json ${outputJson}`);
+    const result = JSON.parse(fs.readFileSync(outputJson, 'utf8'));
+    expect(result.topic_properties).toBeDefined();
+    expect(result.topic_properties['retention.ms']).toBeDefined();
+    expect(result.topic_properties['retention.ms'].property_name).toBe('retention.ms');
+  });
+
+  it('generates AsciiDoc output', () => {
+    execSync(`python3 ${scriptPath} --source-path ${mockSourcePath} --output-adoc ${outputAdoc}`);
+    const adoc = fs.readFileSync(outputAdoc, 'utf8');
+    expect(adoc).toContain('= Topic Configuration Properties');
+    expect(adoc).toContain('retention.ms');
+  });
+});

bin/doc-tools.js

@@ -777,6 +777,41 @@ automation
     process.exit(0);
   });
 
+automation
+  .command('topic-property-docs')
+  .description('Generate JSON and AsciiDoc documentation for Redpanda topic configuration properties')
+  .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
+  .option('--diff <oldTag>', 'Also diff autogenerated topic properties from <oldTag> → <tag>')
+  .action((options) => {
+    verifyPropertyDependencies();
+
+    const newTag = options.tag;
+    const oldTag = options.diff;
+    const cwd = path.resolve(__dirname, '../tools/property-extractor');
+    const make = (tag) => {
+      console.log(`⏳ Building topic property docs for ${tag}…`);
+      const r = spawnSync('make', ['topic-properties', `TAG=${tag}`], { cwd, stdio: 'inherit' });
+      if (r.error) {
+        console.error(`❌ ${r.error.message}`);
+        process.exit(1);
+      }
+      if (r.status !== 0) process.exit(r.status);
+    };
+
+    if (oldTag) {
+      const oldDir = path.join('autogenerated', oldTag, 'properties');
+      if (!fs.existsSync(oldDir)) make(oldTag);
+    }
+
+    make(newTag);
+
+    if (oldTag) {
+      diffDirs('properties', oldTag, newTag);
+    }
+
+    process.exit(0);
+  });
+
 automation
   .command('rpk-docs')
   .description('Generate AsciiDoc documentation for rpk CLI commands')

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.7.2",
+  "version": "4.8.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

tools/property-extractor/Makefile

@@ -1,4 +1,16 @@
-.PHONY: build venv clean redpanda-git treesitter generate-docs check
+.PHONY: build venv clean redpanda-git treesitter topic-properties generate-docs check
+
+# --- Main build: venv, fetch code, build parser, extract & docgen ---
+build: venv redpanda-git treesitter
+	@echo "🔧 Building with Redpanda tag: $(TAG)"
+	@mkdir -p $(TOOL_ROOT)/gen
+	@cd $(TOOL_ROOT) && \
+	  $(PYTHON) -W ignore::FutureWarning property_extractor.py \
+	    --recursive \
+	    --path $(REDPANDA_SRC) \
+	    --output gen/properties-output.json
+	@echo "✅ Cluster properties JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
+	@$(MAKE) generate-docs
 
 # Default tag (can be overridden via `make TAG=v25.1.1`)
 TAG       ?= dev
@@ -25,18 +37,6 @@ PYTHON        := $(VENV)/bin/python
 OUTPUT_DIR    := $(REPO_ROOT)/autogenerated/$(TAG)/properties
 TREE_SITTER   := npx tree-sitter
 
-# --- Main build: venv, fetch code, build parser, extract & docgen ---
-build: venv redpanda-git treesitter
-	@echo "🔧 Building with Redpanda tag: $(TAG)"
-	@mkdir -p $(TOOL_ROOT)/gen
-	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) -W ignore::FutureWarning property_extractor.py \
-	    --recursive \
-	    --path $(REDPANDA_SRC) \
-	    --output gen/properties-output.json
-	@echo "✅ JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
-	@$(MAKE) generate-docs
-
 # --- Ensure Python venv & dependencies ---
 venv: $(TOOL_ROOT)/requirements.txt
 	@if [ ! -d "$(VENV)" ]; then \
@@ -103,3 +103,14 @@ check:
 	@echo "VENV:          $(VENV)"
 	@echo "PYTHON:        $(PYTHON)"
 	@echo "OUTPUT_DIR:    $(OUTPUT_DIR)"
+
+# --- Extract topic properties ---
+topic-properties: venv redpanda-git treesitter
+	@echo "🔧 Extracting topic properties with Redpanda tag: $(TAG)"
+	@mkdir -p $(TOOL_ROOT)/gen
+	@cd $(TOOL_ROOT) && \
+	  $(PYTHON) topic_property_extractor.py \
+	    --source-path $(REDPANDA_SRC) \
+	    --output-json "$(OUTPUT_DIR)/topic-properties-output.json" \
+	    --output-adoc "$(OUTPUT_DIR)/topic-properties.adoc"
+	@echo "✅ Topic properties extracted"

tools/property-extractor/json-to-asciidoc/generate_docs.py

@@ -14,6 +14,7 @@
 OUTPUT_FILE_BROKER = "broker-properties.adoc"
 OUTPUT_FILE_CLUSTER = "cluster-properties.adoc"
 OUTPUT_FILE_CLOUD = "object-storage-properties.adoc"
+OUTPUT_FILE_TOPIC = "topic-properties.adoc"
 OUTPUT_FILE_DEPRECATED = os.path.join("deprecated", "partials", "deprecated-properties.adoc")
 ALL_PROPERTIES_FILE = "all_properties.txt"
 
@@ -66,6 +67,20 @@
 )
 CLUSTER_CONFIG_TITLE = "== Cluster configuration\n\n"
 
+TOPIC_PAGE_TITLE = (
+    "= Topic Configuration Properties\n"
+    ":page-aliases: reference:topic-properties.adoc\n"
+    ":description: Reference of topic configuration properties.\n\n"
+)
+
+TOPIC_INTRO = (
+    "A topic-level property sets a Redpanda or Kafka configuration for a particular topic.\n\n"
+    "Many topic-level properties have corresponding xref:manage:cluster-maintenance/cluster-property-configuration.adoc[cluster properties] that set a default value for all topics of a cluster. To customize the value for a topic, you can set a topic-level property that overrides the value of the corresponding cluster property.\n\n"
+    "NOTE: All topic properties take effect immediately after being set.\n\n"
+)
+
+TOPIC_CONFIG_TITLE = "== Topic configuration\n\n"
+
 CLOUD_PAGE_TITLE = (
     "= Object Storage Properties\n"
     ":description: Reference of object storage properties.\n\n"
@@ -92,7 +107,8 @@
     "src/v/pandaproxy/schema_registry/configuration.cc": "schema reg",
     "src/v/pandaproxy/rest/configuration.cc": "http proxy",
     "src/v/kafka/client/configuration.cc": "http client",
-    "src/v/config/configuration.cc": "cluster"
+    "src/v/config/configuration.cc": "cluster",
+    "src/v/kafka/server/handlers/topics/types.cc": "topic"
 }
 
 SUFFIX_TO_UNIT = {
@@ -339,6 +355,7 @@ def main():
     kafka_client_content = []
     cluster_config_content = []
     cloud_config_content = []
+    topic_config_content = []
     deprecated_broker_content = []
     deprecated_cluster_content = []
     all_properties = []
@@ -388,6 +405,7 @@ def main():
             "http client": kafka_client_content,
             "cluster": cluster_config_content,
             "cloud": cloud_config_content,
+            "topic": topic_config_content,
         }
         if group in group_mapping:
             group_mapping[group].append(property_doc)
@@ -423,6 +441,12 @@ def main():
         + CLOUD_CONFIG_TITLE
         + "".join(cloud_config_content)
     )
+    topic_page = (
+        TOPIC_PAGE_TITLE
+        + TOPIC_INTRO
+        + TOPIC_CONFIG_TITLE
+        + "".join(topic_config_content)
+    )
     deprecated_page = (
         DEPRECATED_PROPERTIES_TITLE
         + DEPRECATED_PROPERTIES_INTRO
@@ -436,6 +460,7 @@ def main():
     write_data_to_file(page_folder, OUTPUT_FILE_BROKER, broker_page)
     write_data_to_file(page_folder, OUTPUT_FILE_CLUSTER, cluster_page)
     write_data_to_file(page_folder, OUTPUT_FILE_CLOUD, cloud_page)
+    write_data_to_file(page_folder, OUTPUT_FILE_TOPIC, topic_page)
     write_data_to_file(page_folder, OUTPUT_FILE_DEPRECATED, deprecated_page)
     write_data_to_file(output_dir, ALL_PROPERTIES_FILE, "\n".join(all_properties))