Skip to content

Commit 990d436

Browse files
committed
feat(datastore): add SQLite database backend support
Signed-off-by: Chanakya TS <[email protected]>
1 parent a1b414d commit 990d436

File tree

57 files changed

+1433
-5
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+1433
-5
lines changed

.github/workflows/check-db-schema-structs.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,23 @@ jobs:
5151
git diff
5252
exit 1
5353
fi
54+
check-sqlite-schema-structs:
55+
runs-on: ubuntu-latest
56+
steps:
57+
- uses: actions/checkout@v5
58+
- name: Setup Go
59+
uses: actions/setup-go@v5
60+
with:
61+
go-version: "1.24.4"
62+
- name: Generate SQLite DB schema structs
63+
run: make gen/gorm/sqlite
64+
- name: Check if there are uncommitted file changes
65+
run: |
66+
clean=$(git status --porcelain)
67+
if [[ -z "$clean" ]]; then
68+
echo "SQLite schema is up to date."
69+
else
70+
echo "Uncommitted file changes detected after generating SQLite schema: $clean"
71+
git diff
72+
exit 1
73+
fi

Makefile

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,16 @@ start/postgres:
114114
stop/postgres:
115115
./scripts/teardown_postgres_db.sh
116116

117+
# Start the SQLite database (file-based, minimal setup)
118+
.PHONY: start/sqlite
119+
start/sqlite:
120+
./scripts/start_sqlite_db.sh
121+
122+
# Stop the SQLite database (cleanup)
123+
.PHONY: stop/sqlite
124+
stop/sqlite:
125+
./scripts/teardown_sqlite_db.sh
126+
117127
# generate the gorm structs for MySQL
118128
.PHONY: gen/gorm/mysql
119129
gen/gorm/mysql: bin/golang-migrate start/mysql
@@ -129,12 +139,23 @@ gen/gorm/postgres: bin/golang-migrate start/postgres
129139
cd gorm-gen && GOWORK=off go run main.go --db-type postgres --dsn 'postgres://postgres:postgres@localhost:5432/model-registry?sslmode=disable' && \
130140
cd $(CURDIR) && ./scripts/remove_gorm_defaults.sh)
131141

142+
# generate the gorm structs for SQLite
143+
.PHONY: gen/gorm/sqlite
144+
gen/gorm/sqlite: bin/golang-migrate start/sqlite
145+
@(trap 'cd $(CURDIR) && $(MAKE) stop/sqlite' EXIT; \
146+
$(GOLANG_MIGRATE) -path './internal/datastore/embedmd/sqlite/migrations' -database 'sqlite://./model-registry.db' up && \
147+
cd gorm-gen && GOWORK=off go run main.go --db-type sqlite --dsn './model-registry.db' && \
148+
cd $(CURDIR) && ./scripts/remove_gorm_defaults.sh)
149+
132150
# generate the gorm structs (defaults to MySQL for backward compatibility)
133151
# Use GORM_DB_TYPE=postgres to generate for PostgreSQL instead
152+
# Use GORM_DB_TYPE=sqlite to generate for SQLite instead
134153
.PHONY: gen/gorm
135154
gen/gorm: bin/golang-migrate
136155
ifeq ($(GORM_DB_TYPE),postgres)
137156
$(MAKE) gen/gorm/postgres
157+
else ifeq ($(GORM_DB_TYPE),sqlite)
158+
$(MAKE) gen/gorm/sqlite
138159
else
139160
$(MAKE) gen/gorm/mysql
140161
endif
@@ -185,7 +206,7 @@ bin/yq:
185206

186207
GOLANG_MIGRATE ?= ${PROJECT_BIN}/migrate
187208
bin/golang-migrate:
188-
GOBIN=$(PROJECT_PATH)/bin ${GO} install -tags 'mysql,postgres' github.com/golang-migrate/migrate/v4/cmd/[email protected]
209+
GOBIN=$(PROJECT_PATH)/bin ${GO} install -tags 'mysql,postgres,sqlite' github.com/golang-migrate/migrate/v4/cmd/[email protected]
189210

190211
GENQLIENT ?= ${PROJECT_BIN}/genqlient
191212
bin/genqlient:

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ require (
2828
google.golang.org/protobuf v1.36.8
2929
gorm.io/driver/mysql v1.6.0
3030
gorm.io/driver/postgres v1.6.0
31+
gorm.io/driver/sqlite v1.6.0
3132
gorm.io/gorm v1.30.1
3233
k8s.io/api v0.33.4
3334
k8s.io/apimachinery v0.33.4
@@ -102,6 +103,7 @@ require (
102103
github.com/leodido/go-urn v1.4.0 // indirect
103104
github.com/lib/pq v1.10.9 // indirect
104105
github.com/mailru/easyjson v0.9.0 // indirect
106+
github.com/mattn/go-sqlite3 v1.14.22 // indirect
105107
github.com/moby/docker-image-spec v1.3.1 // indirect
106108
github.com/moby/go-archive v0.1.0 // indirect
107109
github.com/moby/sys/user v0.4.0 // indirect

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,8 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S
303303
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
304304
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
305305
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
306+
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
307+
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
306308
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
307309
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
308310
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
@@ -661,6 +663,8 @@ gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
661663
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
662664
gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
663665
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
666+
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
667+
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
664668
gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4=
665669
gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
666670
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=

go.work.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,8 @@ gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKK
662662
gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI=
663663
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
664664
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
665+
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
666+
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
665667
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
666668
istio.io/api v1.24.2 h1:jYjcN6Iq0RPtQj/3KMFsybxmfqmjGN/dxhL7FGJEdIM=
667669
istio.io/api v1.24.2/go.mod h1:MQnRok7RZ20/PE56v0LxmoWH0xVxnCQPNuf9O7PAN1I=

internal/datastore/embedmd/service.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ type EmbedMDConfig struct {
2020
}
2121

2222
func (c *EmbedMDConfig) Validate() error {
23-
if c.DatabaseType != types.DatabaseTypeMySQL && c.DatabaseType != types.DatabaseTypePostgres {
24-
return fmt.Errorf("unsupported database type: %s. Supported types: %s, %s", c.DatabaseType, types.DatabaseTypeMySQL, types.DatabaseTypePostgres)
23+
if c.DatabaseType != types.DatabaseTypeMySQL && c.DatabaseType != types.DatabaseTypePostgres && c.DatabaseType != types.DatabaseTypeSQLite {
24+
return fmt.Errorf("unsupported database type: %s. Supported types: %s, %s, %s", c.DatabaseType, types.DatabaseTypeMySQL, types.DatabaseTypePostgres, types.DatabaseTypeSQLite)
2525
}
2626

2727
return nil
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package sqlite
2+
3+
import (
4+
"fmt"
5+
"sync"
6+
"time"
7+
8+
"github.com/golang/glog"
9+
_tls "github.com/kubeflow/model-registry/internal/tls"
10+
"gorm.io/driver/sqlite"
11+
"gorm.io/gorm"
12+
"gorm.io/gorm/logger"
13+
)
14+
15+
const (
16+
// sqliteMaxRetriesDefault is the maximum number of attempts to retry SQLite connection.
17+
sqliteMaxRetriesDefault = 5 // SQLite is file-based, so fewer retries needed
18+
)
19+
20+
type SQLiteDBConnector struct {
21+
DSN string
22+
TLSConfig *_tls.TLSConfig // Not used for SQLite but kept for interface consistency
23+
db *gorm.DB
24+
connectMutex sync.Mutex
25+
maxRetries int
26+
}
27+
28+
func NewSQLiteDBConnector(
29+
dsn string,
30+
tlsConfig *_tls.TLSConfig,
31+
) *SQLiteDBConnector {
32+
return &SQLiteDBConnector{
33+
DSN: dsn,
34+
TLSConfig: tlsConfig,
35+
maxRetries: sqliteMaxRetriesDefault,
36+
}
37+
}
38+
39+
func (c *SQLiteDBConnector) WithMaxRetries(maxRetries int) *SQLiteDBConnector {
40+
c.maxRetries = maxRetries
41+
42+
return c
43+
}
44+
45+
func (c *SQLiteDBConnector) Connect() (*gorm.DB, error) {
46+
// Use mutex to ensure only one connection attempt at a time
47+
c.connectMutex.Lock()
48+
defer c.connectMutex.Unlock()
49+
50+
// If we already have a working connection, return it
51+
if c.db != nil {
52+
return c.db, nil
53+
}
54+
55+
var db *gorm.DB
56+
var err error
57+
58+
// Log warning if TLS configuration is specified (not supported by SQLite)
59+
if c.TLSConfig != nil && c.needsTLSConfig() {
60+
glog.Warningf("TLS configuration is not supported for SQLite connections, ignoring TLS settings")
61+
}
62+
63+
for i := range c.maxRetries {
64+
glog.V(2).Infof("Attempting to connect to SQLite database: %q (attempt %d/%d)", c.DSN, i+1, c.maxRetries)
65+
db, err = gorm.Open(sqlite.Open(c.DSN), &gorm.Config{
66+
Logger: logger.Default.LogMode(logger.Silent),
67+
TranslateError: true,
68+
})
69+
if err == nil {
70+
break
71+
}
72+
73+
glog.Warningf("Retrying connection to SQLite (attempt %d/%d): %v", i+1, c.maxRetries, err)
74+
75+
time.Sleep(time.Duration(i+1) * time.Second)
76+
}
77+
78+
if err != nil {
79+
return nil, fmt.Errorf("failed to connect to SQLite: %w", err)
80+
}
81+
82+
glog.Info("Successfully connected to SQLite database")
83+
84+
c.db = db
85+
86+
return db, nil
87+
}
88+
89+
func (c *SQLiteDBConnector) DB() *gorm.DB {
90+
return c.db
91+
}
92+
93+
func (c *SQLiteDBConnector) needsTLSConfig() bool {
94+
if c.TLSConfig == nil {
95+
return false
96+
}
97+
98+
return c.TLSConfig.CertPath != "" || c.TLSConfig.KeyPath != "" || c.TLSConfig.RootCertPath != "" || c.TLSConfig.CAPath != "" || c.TLSConfig.Cipher != "" || c.TLSConfig.VerifyServerCert
99+
}

0 commit comments

Comments
 (0)