PMM-14368 PostgreSQL 18 support

.github/workflows/go.yml

@@ -14,11 +14,12 @@ jobs:
     strategy:
       matrix:
         postgresql-image:
-          - postgres:10
-          - postgres:11
-          - postgres:12
           - postgres:13
           - postgres:14
+          - postgres:15
+          - postgres:16
+          - postgres:17
+          - postgres:18
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code

CHANGELOG.md

@@ -1,3 +1,12 @@
+## [Unreleased]
+
+* [ENHANCEMENT] Add PostgreSQL 18 support:
+  * Add parallel worker activity metrics
+  * Add vacuum/analyze timing metrics
+  * Add enhanced checkpointer metrics
+  * Add `pg_stat_io` collector with byte statistics and WAL I/O activity tracking
+* [ENHANCEMENT] Update CI tested PostgreSQL versions to include PostgreSQL 18
+
 ## 0.15.0 / 2023-10-27
 
 * [ENHANCEMENT] Add 1kB and 2kB units #915

README.md

@@ -7,7 +7,7 @@
 
 Prometheus exporter for PostgreSQL server metrics.
 
-CI Tested PostgreSQL versions: `11`, `12`, `13`, `14`, `15`, `16`
+CI Tested PostgreSQL versions: `13`, `14`, `15`, `16`, `18`
 
 ## Quick Start
 This package is available for Docker:

collector/pg_stat_bgwriter.go

@@ -16,6 +16,7 @@ package collector
 import (
 	"context"
 	"database/sql"
+
 	"github.com/blang/semver/v4"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -46,6 +47,12 @@ var (
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	)
+	statBGWriterCheckpointsDoneDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoints_done_total"),
+		"Number of completed checkpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	)
 	statBGWriterCheckpointsReqTimeDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoint_write_time_total"),
 		"Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds",
@@ -94,6 +101,12 @@ var (
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	)
+	statBGWriterCheckpointsSlruWrittenDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "slru_written_total"),
+		"Number of SLRU buffers written during checkpoints and restartpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	)
 	statBGWriterStatsResetDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "stats_reset_total"),
 		"Time at which these statistics were last reset",
@@ -114,6 +127,12 @@ var statBGWriter = map[string]*prometheus.Desc{
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	),
+	"percona_checkpoints_done": prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoints_done"),
+		"Number of completed checkpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	),
 	"percona_checkpoint_write_time": prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoint_write_time"),
 		"Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds",
@@ -162,6 +181,12 @@ var statBGWriter = map[string]*prometheus.Desc{
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	),
+	"percona_slru_written": prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "slru_written"),
+		"Number of SLRU buffers written during checkpoints and restartpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	),
 	"percona_stats_reset": prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "stats_reset"),
 		"Time at which these statistics were last reset",
@@ -194,22 +219,30 @@ const statBGWriterQueryPost17 = `SELECT
 const statCheckpointerQuery = `SELECT
 		num_timed
 		,num_requested
+		CASE WHEN current_setting('server_version_num')::int >= 180000
+			 THEN COALESCE(num_done, 0)
+			 ELSE 0 END as num_done,
 		,restartpoints_timed
 		,restartpoints_req
 		,restartpoints_done
 		,write_time
 		,sync_time
 		,buffers_written
+		,CASE WHEN current_setting('server_version_num')::int >= 180000
+			 THEN COALESCE(slru_written, 0)
+			 ELSE 0 END as slru_written,
 		,stats_reset
 	FROM pg_stat_checkpointer;`
 
 func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error {
 	db := instance.getDB()
 
-	var cpt, cpr, bcp, bc, mwc, bb, bbf, ba sql.NullInt64
+	var cpt, cpr, cpd, bcp, bc, mwc, bb, bbf, ba, slruw sql.NullInt64
 	var cpwt, cpst sql.NullFloat64
 	var sr sql.NullTime
 
+	after18 := instance.version.GTE(semver.Version{Major: 18})
+
 	if instance.version.GE(semver.MustParse("17.0.0")) {
 		row := db.QueryRowContext(ctx,
 			statBGWriterQueryPost17)
@@ -219,10 +252,9 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		}
 		var rpt, rpr, rpd sql.NullInt64
 		var csr sql.NullTime
-		// these variables are not used, but I left them here for reference
-		row = db.QueryRowContext(ctx,
-			statCheckpointerQuery)
-		err = row.Scan(&cpt, &cpr, &rpt, &rpr, &rpd, &cpwt, &cpst, &bcp, &csr)
+
+		row = db.QueryRowContext(ctx, statCheckpointerQuery)
+		err = row.Scan(&cpt, &cpr, &cpd, &rpt, &rpr, &rpd, &cpwt, &cpst, &bcp, &slruw, &csr)
 		if err != nil {
 			return err
 		}
@@ -257,6 +289,21 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+
+	cpdMetric := 0.0
+	if after18 {
+		if cpd.Valid {
+			cpdMetric = float64(cpd.Int64)
+		}
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriterCheckpointsDoneDesc,
+			prometheus.CounterValue,
+			cpdMetric,
+			"exporter",
+			instance.name,
+		)
+	}
+
 	cpwtMetric := 0.0
 	if cpwt.Valid {
 		cpwtMetric = float64(cpwt.Float64)
@@ -345,6 +392,19 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+	slruwMetric := 0.0
+	if after18 {
+		if slruw.Valid {
+			slruwMetric = float64(slruw.Int64)
+		}
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriterCheckpointsSlruWrittenDesc,
+			prometheus.CounterValue,
+			slruwMetric,
+			"exporter",
+			instance.name,
+		)
+	}
 	srMetric := 0.0
 	if sr.Valid {
 		srMetric = float64(sr.Time.Unix())
@@ -373,6 +433,15 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+	if after18 {
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriter["percona_checkpoints_done"],
+			prometheus.CounterValue,
+			cpdMetric,
+			"exporter",
+			instance.name,
+		)
+	}
 	ch <- prometheus.MustNewConstMetric(
 		statBGWriter["percona_checkpoint_write_time"],
 		prometheus.CounterValue,
@@ -429,6 +498,15 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+	if after18 {
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriter["percona_slru_written"],
+			prometheus.CounterValue,
+			slruwMetric,
+			"exporter",
+			instance.name,
+		)
+	}
 	ch <- prometheus.MustNewConstMetric(
 		statBGWriter["percona_stats_reset"],
 		prometheus.CounterValue,

collector/pg_stat_database.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"database/sql"
 
+	"github.com/blang/semver/v4"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -206,6 +207,24 @@ var (
 		[]string{"datid", "datname"},
 		prometheus.Labels{},
 	)
+	statDatabaseParallelWorkersToLaunch = prometheus.NewDesc(prometheus.BuildFQName(
+		namespace,
+		statDatabaseSubsystem,
+		"parallel_workers_to_launch",
+	),
+		"Number of parallel workers to launch (PostgreSQL 18+)",
+		[]string{"datid", "datname"},
+		prometheus.Labels{},
+	)
+	statDatabaseParallelWorkersLaunched = prometheus.NewDesc(prometheus.BuildFQName(
+		namespace,
+		statDatabaseSubsystem,
+		"parallel_workers_launched",
+	),
+		"Number of parallel workers launched (PostgreSQL 18+)",
+		[]string{"datid", "datname"},
+		prometheus.Labels{},
+	)
 
 	statDatabaseQuery = `
 		SELECT
@@ -228,15 +247,22 @@ var (
 			,blk_read_time
 			,blk_write_time
 			,stats_reset
+			CASE WHEN current_setting('server_version_num')::int >= 180000
+				THEN parallel_workers_to_launch
+				ELSE NULL END as parallel_workers_to_launch
+			CASE WHEN current_setting('server_version_num')::int >= 180000
+				THEN parallel_workers_launched
+				ELSE NULL END as parallel_workers_launched
 		FROM pg_stat_database;
 	`
 )
 
 func (c *PGStatDatabaseCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error {
 	db := instance.getDB()
-	rows, err := db.QueryContext(ctx,
-		statDatabaseQuery,
-	)
+
+	after18 := instance.version.GTE(semver.Version{Major: 18})
+
+	rows, err := db.QueryContext(ctx, statDatabaseQuery)
 	if err != nil {
 		return err
 	}
@@ -246,6 +272,7 @@ func (c *PGStatDatabaseCollector) Update(ctx context.Context, instance *instance
 		var datid, datname sql.NullString
 		var numBackends, xactCommit, xactRollback, blksRead, blksHit, tupReturned, tupFetched, tupInserted, tupUpdated, tupDeleted, conflicts, tempFiles, tempBytes, deadlocks, blkReadTime, blkWriteTime sql.NullFloat64
 		var statsReset sql.NullTime
+		var parallelWorkersToLaunch, parallelWorkersLaunched sql.NullFloat64
 
 		err := rows.Scan(
 			&datid,
@@ -267,6 +294,8 @@ func (c *PGStatDatabaseCollector) Update(ctx context.Context, instance *instance
 			&blkReadTime,
 			&blkWriteTime,
 			&statsReset,
+			&parallelWorkersToLaunch,
+			&parallelWorkersLaunched,
 		)
 		if err != nil {
 			return err
@@ -353,6 +382,18 @@ func (c *PGStatDatabaseCollector) Update(ctx context.Context, instance *instance
 			statsResetMetric = float64(statsReset.Time.Unix())
 		}
 
+		if after18 {
+			if !parallelWorkersToLaunch.Valid {
+				level.Debug(c.log).Log("msg", "Skipping collecting metric because it has no parallel_workers_to_launch")
+				continue
+			}
+
+			if !parallelWorkersLaunched.Valid {
+				level.Debug(c.log).Log("msg", "Skipping collecting metric because it has no parallel_workers_launched")
+				continue
+			}
+		}
+
 		labels := []string{datid.String, datname.String}
 
 		ch <- prometheus.MustNewConstMetric(
@@ -473,6 +514,22 @@ func (c *PGStatDatabaseCollector) Update(ctx context.Context, instance *instance
 			statsResetMetric,
 			labels...,
 		)
+
+		if after18 {
+			ch <- prometheus.MustNewConstMetric(
+				statDatabaseParallelWorkersToLaunch,
+				prometheus.CounterValue,
+				parallelWorkersToLaunch.Float64,
+				labels...,
+			)
+
+			ch <- prometheus.MustNewConstMetric(
+				statDatabaseParallelWorkersLaunched,
+				prometheus.CounterValue,
+				parallelWorkersLaunched.Float64,
+				labels...,
+			)
+		}
 	}
 	return nil
 }