Add Iceberg support in test cluster

Author: JakeSCahill

docker-compose/bootstrap.yml

@@ -48,9 +48,18 @@ enable_consumer_group_metrics:
   - "group"
   - "partition"
   - "consumer_lag"
-# Lower the interval for the quickstart
+# Lower the interval for the autogeneration of consumer group metrics.
 # https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
 consumer_group_lag_collection_interval_sec: 60
 # Enable Redpanda to collect host metrics.
 # https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
-enable_host_metrics: true
+enable_host_metrics: true
+# Enable for Iceberg metrics
+iceberg_enabled: true
+# Set up Iceberg REST catalog configuration
+iceberg_catalog_type: rest
+iceberg_rest_catalog_endpoint: http://catalog:8181
+# Credentials are required, but the catalog ignores them
+iceberg_rest_catalog_client_id: catalog
+iceberg_rest_catalog_client_secret: catalog123
+iceberg_catalog_commit_interval_ms: 5000

docker-compose/docker-compose.yml

@@ -1,4 +1,4 @@
-name: redpanda-quickstart-multi-broker
+name: redpanda-cluster
 networks:
   redpanda_network:
     driver: bridge
@@ -265,6 +265,7 @@ services:
       - -X user=superuser
       - -X pass=secretpassword
       - -X brokers=redpanda-0:9092
+      - --topic-config=redpanda.iceberg.mode=key_value
     image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
     networks:
       - redpanda_network
@@ -394,3 +395,19 @@ services:
       /usr/bin/mc policy set public minio/redpanda;
       tail -f /dev/null
       "
+  catalog:
+    image: tabulario/iceberg-rest
+    container_name: iceberg-rest
+    networks:
+      - redpanda_network
+    depends_on:
+      - minio
+    ports:
+      - 8181:8181
+    environment:
+      - AWS_ACCESS_KEY_ID=minio
+      - AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
+      - AWS_REGION=local
+      - CATALOG_WAREHOUSE=s3://redpanda/
+      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+      - CATALOG_S3_ENDPOINT=http://minio:9000

tools/metrics/metrics.py

@@ -35,67 +35,78 @@ def fetch_metrics(url):
 
 def parse_metrics(metrics_text):
     """
-    Parse Prometheus exposition text into a structured dict of:
+    Parse Prometheus exposition text into a dict:
       metric_name → { description, type, labels: [<label_keys>] }
-    Robust against any ordering of # HELP/# TYPE and handles samples
-    both with and without labels.
+
+    - Strips empty `{}` on unlabelled samples
+    - Propagates HELP/TYPE from base metrics onto _bucket, _count, _sum
+    - Works regardless of # HELP / # TYPE order
     """
-    import re
-    import logging
+    import re, logging
 
-    # Gather HELP/type metadata
-    meta = {}      # name → { 'description':…, 'type':… }
     lines = metrics_text.splitlines()
+
+    # Gather HELP/TYPE metadata in any order
+    meta = {}  # name → { 'description': str, 'type': str }
     for line in lines:
         if line.startswith("# HELP"):
             m = re.match(r"# HELP\s+(\S+)\s+(.+)", line)
             if m:
-                name, desc = m.group(1), m.group(2)
+                name, desc = m.groups()
                 meta.setdefault(name, {})['description'] = desc
         elif line.startswith("# TYPE"):
             m = re.match(r"# TYPE\s+(\S+)\s+(\S+)", line)
             if m:
-                name, t = m.group(1), m.group(2)
-                meta.setdefault(name, {})['type'] = t
+                name, mtype = m.groups()
+                meta.setdefault(name, {})['type'] = mtype
 
-    # Collect label keys from every sample line
+    # Collect label keys from _every_ sample line
     label_map = {}  # name → set(label_keys)
     for line in lines:
         if line.startswith("#"):
             continue
 
-        # try labelled sample: metric{a="1",b="2"} 42
+        # labelled: foo{a="1",b="2"}  42
         m_lbl = re.match(r"^(\S+)\{(.+?)\}\s+(.+)", line)
         if m_lbl:
-            name, labels_str = m_lbl.group(1), m_lbl.group(2)
-            keys = [kv.split("=", 1)[0] for kv in labels_str.split(",")]
+            name, labels_str, _ = m_lbl.groups()
+            keys = [kv.split("=",1)[0] for kv in labels_str.split(",")]
         else:
-            # fallback to unlabelled: metric 42
-            m_unlbl = re.match(r"^(\S+)\s+(.+)", line)
-            if m_unlbl:
-                name, keys = m_unlbl.group(1), []
-            else:
+            # unlabelled, maybe with stray {}: foo{} 42  or just foo 42
+            m_unlbl = re.match(r"^(\S+?)(?:\{\})?\s+(.+)", line)
+            if not m_unlbl:
                 continue
+            name, _ = m_unlbl.groups()
+            keys = []
 
         label_map.setdefault(name, set()).update(keys)
 
-    # Merge into final structure, warn on missing pieces
+    # Propagate HELP/TYPE from base histograms/summaries
+    for series in list(label_map):
+        for suffix in ("_bucket", "_count", "_sum"):
+            if series.endswith(suffix):
+                base = series[:-len(suffix)]
+                if base in meta:
+                    meta.setdefault(series, {}).update(meta[base])
+                break
+
+    # Merge into final metrics dict, with warnings
     metrics = {}
     all_names = set(meta) | set(label_map)
     for name in sorted(all_names):
         desc = meta.get(name, {}).get("description")
-        t    = meta.get(name, {}).get("type")
+        mtype = meta.get(name, {}).get("type")
         labels = sorted(label_map.get(name, []))
 
         if desc is None:
             logging.warning(f"Metric '{name}' has samples but no # HELP.")
             desc = ""
-        if t is None:
+        if mtype is None:
             logging.warning(f"Metric '{name}' has no # TYPE entry.")
 
         metrics[name] = {
             "description": desc,
-            "type": t,
+            "type": mtype,
             "labels": labels
         }
 

tools/property-extractor/property_extractor.py

@@ -85,12 +85,11 @@ def get_files_with_properties(file_pairs, treesitter_parser, cpp_language):
 
 
 def transform_files_with_properties(files_with_properties):
-    type_transformer = TypeTransformer()
     transformers = [
+        TypeTransformer(),
         EnterpriseTransformer(), ## this must be the first, as it modifies current data
         MetaParamTransformer(),
         BasicInfoTransformer(),
-        type_transformer,
         IsNullableTransformer(),
         IsArrayTransformer(type_transformer),
         NeedsRestartTransformer(),