Use optimized checksumming when loading database (#441)

Author: btoews

.github/workflows/push.yml

@@ -41,7 +41,10 @@ jobs:
           go-version-file: 'go.mod'
 
       - name: apt install
-        run: sudo apt install -y fuse3 libsqlite3-dev consul
+        run: |
+          wget -O - https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y fuse3 libsqlite3-dev consul
 
       - name: check fuse version
         run: fusermount -V
@@ -75,7 +78,10 @@ jobs:
           go-version-file: 'go.mod'
 
       - name: apt install
-        run: sudo apt install -y fuse3 libsqlite3-dev consul
+        run: |
+          wget -O - https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y fuse3 libsqlite3-dev consul
 
       - name: Start consul in dev mode
         run: consul agent -dev &

cmd/litefs/export_test.go

@@ -1,4 +1,5 @@
-// go:build linux
+//go:build linux
+
 package main_test
 
 import (

cmd/litefs/import_test.go

@@ -1,4 +1,5 @@
-// go:build linux
+//go:build linux
+
 package main_test
 
 import (

cmd/litefs/mount_linux.go

@@ -1,4 +1,5 @@
-// go:build linux
+//go:build linux
+
 package main
 
 import (

cmd/litefs/mount_test.go

@@ -1,4 +1,5 @@
-// go:build linux
+//go:build linux
+
 package main_test
 
 import (

db.go

@@ -930,20 +930,26 @@ func (db *DB) initDatabaseFile() error {
 	// Build per-page checksum map for existing pages. The database could be
 	// short compared to the page count in the header so just checksum what we
 	// can. The database may recover in applyLTX() so we'll do validation then.
-	buf := make([]byte, db.pageSize)
 	db.chksums.pages = make([]ltx.Checksum, db.PageN())
 	db.chksums.blocks = make([]ltx.Checksum, pageChksumBlock(db.PageN()))
-	for pgno := uint32(1); pgno <= db.PageN(); pgno++ {
-		offset := int64(pgno-1) * int64(db.pageSize)
-		if _, err := internal.ReadFullAt(f, buf, offset); err == io.EOF || err == io.ErrUnexpectedEOF {
-			log.Printf("database checksum ending early at page %d of %d ", pgno-1, db.PageN())
-			break
-		} else if err != nil {
-			return fmt.Errorf("read database page %d: %w", pgno, err)
-		}
 
-		chksum := ltx.ChecksumPage(pgno, buf)
-		db.setDatabasePageChecksum(pgno, chksum)
+	lastGoodPage, err := ltx.ChecksumPages(db.DatabasePath(), db.pageSize, db.PageN(), 0, db.chksums.pages)
+
+	// lastGoodPage tells us how far we got before the first error. Most likely
+	// is that we got an EOF because the db was short, in which case no
+	// subsequent checksums would have been written. To be cautious though,
+	// zero out any checksums after the last good page.
+	clear(db.chksums.pages[lastGoodPage:])
+
+	// Always overwrite the lock page as a zero checksum.
+	if lockPage := ltx.LockPgno(db.pageSize); len(db.chksums.pages) >= int(lockPage) {
+		db.chksums.pages[lockPage-1] = 0
+	}
+
+	if err == io.EOF || err == io.ErrUnexpectedEOF {
+		log.Printf("database checksum ending early at page %d of %d ", lastGoodPage, db.PageN())
+	} else if err != nil {
+		return fmt.Errorf("read database page %d: %w", lastGoodPage+1, err)
 	}
 
 	return nil

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.16-0.20220918133448-90900be5db1a
 	github.com/prometheus/client_golang v1.13.0
 	github.com/superfly/litefs-go v0.0.0-20230227231337-34ea5dcf1e0b
-	github.com/superfly/ltx v0.3.13
+	github.com/superfly/ltx v0.3.14
 	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
 	golang.org/x/net v0.17.0
 	golang.org/x/sync v0.4.0

go.sum

@@ -308,8 +308,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/superfly/litefs-go v0.0.0-20230227231337-34ea5dcf1e0b h1:+WuhtZFB8fNdPeaMUtuB/U8aknXBXdDW/mBm/HTYJNg=
 github.com/superfly/litefs-go v0.0.0-20230227231337-34ea5dcf1e0b/go.mod h1:h+GUx1V2s0C5nY73ZN82760eWEJrpMaiDweF31VmJKk=
-github.com/superfly/ltx v0.3.13 h1:IbuocKJ6sY2jYvZbpUGMYmTkvaLSGUderEZwmaIUmJ0=
-github.com/superfly/ltx v0.3.13/go.mod h1:ly+Dq7UVacQVEI5/b0r6j+PSNy9ibwx1yikcWAaSkhE=
+github.com/superfly/ltx v0.3.14 h1:u/w2NJsyaAyh04+Oy2x5tQpWJzpzf9iQbY1wgfMZnW4=
+github.com/superfly/ltx v0.3.14/go.mod h1:ly+Dq7UVacQVEI5/b0r6j+PSNy9ibwx1yikcWAaSkhE=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=