diff --git a/docs/upgrade-k8s-1.33.md b/docs/upgrade-k8s-1.33.md index 3a9ede3..58e1575 100644 --- a/docs/upgrade-k8s-1.33.md +++ b/docs/upgrade-k8s-1.33.md @@ -230,8 +230,31 @@ Findings from 2026-10-23: - [ ] `sudo snap refresh microk8s --channel=1.36/stable`, addons repo, Calico ### Shrink the database -- [ ] On dqlite 1.18.2+ (so from microk8s 1.34): offline backup, then a - supervised VACUUM (the maintainers advise against running it - unattended). Expect the file to go from ~709 MB to ~35 MB and snapshots - to shrink with it + +dqlite accepts VACUUM since v1.18.2 (canonical/dqlite#734). Its maintainer: +"it potentially create a very big transaction which has effects both on the +RAFT protocol *and* on resource usage. I would recommend not to run it +unsupervised." The dqlite shell that ships with microk8s can't send it: it +wraps every statement in a transaction, and SQLite refuses to VACUUM inside +one. `tools/dqlite-vacuum` sends it through the go-dqlite driver as a plain +statement, as user `ruben` (in the `microk8s` group, which can read +`cluster.crt`, `cluster.key` and `info.yaml`); without `-vacuum` it only +prints page statistics. + +Rehearsed on 2026-10-23 on opti1 with the k8s-dqlite and dqlite binaries +of microk8s 1.34, on a throwaway node in `/dev/shm` (not in Docker on the +Mac: the amd64 emulation lacks `io_setup`, which raft needs): 657 MB with +616 MB free went to 41 MB with none free in 0.6s; a writer next to it saw +no failures and no write over 1s; after a restart, and after a restart from +the first snapshot (40 MB), `integrity_check` was ok and all rows were +there. + +Live database before: 709 MB, 666 MB free, ~4,400 kine rows; snapshots of +~150 MB every ~23 minutes; k8s-dqlite RSS 1,346 MB. + +- [ ] `ssh -t ruben@192.168.1.204 ./dqlite-vacuum.sh`: dbctl backup, an + offline copy of the backend (API down ~1 min), VACUUM, integrity check. + How to go back is at the top of the script +- [ ] Afterwards: the next snapshot's size (expect ~8 MB instead of + ~150 MB), lease update times, controller restarts, k8s-dqlite memory diff --git a/tools/dqlite-vacuum/dqlite-vacuum.sh b/tools/dqlite-vacuum/dqlite-vacuum.sh new file mode 100644 index 0000000..e7050c8 --- /dev/null +++ b/tools/dqlite-vacuum/dqlite-vacuum.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# VACUUM the k8s-dqlite database, see "Shrink the database" in +# docs/upgrade-k8s-1.33.md. Copy this script and the tool next to it to the +# home directory on opti1; build the tool with +# CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dqlite-vacuum . +# +# Rehearsed on a throwaway node with the binaries of microk8s 1.34: 657 MB to +# 41 MB in 0.6s, writes next to it kept working, and the data was intact +# after a restart and after the first snapshot. +# +# To go back to the state before the VACUUM: +# sudo snap stop microk8s.daemon-k8s-dqlite +# cd /var/snap/microk8s/current/var/kubernetes +# sudo mv backend backend.vacuumed && sudo tar -xpf /root/backend-pre-vacuum-.tar +# sudo snap start microk8s.daemon-k8s-dqlite +set -euo pipefail +stamp=$(date +%Y%m%d-%H%M) +kube=/var/snap/microk8s/current/var/kubernetes +shell="/snap/microk8s/current/bin/dqlite -s 127.0.0.1:19001 -c $kube/backend/cluster.crt -k $kube/backend/cluster.key --timeout 60000 k8s" +echo "== $(date) start" + +echo "== datastore backup to /root/dqlite-pre-vacuum-$stamp.tar.gz" +sudo microk8s dbctl backup -o /root/dqlite-pre-vacuum-$stamp + +echo "== offline copy of the backend (API down ~1 min, pods keep running)" +sudo snap stop microk8s.daemon-k8s-dqlite +sudo tar -cpf /root/backend-pre-vacuum-$stamp.tar -C $kube backend +sudo snap start microk8s.daemon-k8s-dqlite +sudo ls -la /root/backend-pre-vacuum-$stamp.tar /root/dqlite-pre-vacuum-$stamp.tar.gz +sudo microk8s status --wait-ready --timeout 300 >/dev/null && echo "microk8s is ready" +echo "== letting the API server and controllers settle for a minute" +sleep 60 + +echo "== VACUUM" +ps -o rss= -C k8s-dqlite | awk '{printf "k8s-dqlite RSS before: %.0f MB\n", $1/1024}' +~/dqlite-vacuum -vacuum +echo "== integrity check" +$shell "PRAGMA integrity_check" +ps -o rss= -C k8s-dqlite | awk '{printf "k8s-dqlite RSS after: %.0f MB\n", $1/1024}' +echo "== $(date) done" diff --git a/tools/dqlite-vacuum/go.mod b/tools/dqlite-vacuum/go.mod new file mode 100644 index 0000000..05baee4 --- /dev/null +++ b/tools/dqlite-vacuum/go.mod @@ -0,0 +1,16 @@ +module dqlite-vacuum + +go 1.27.3 + +require ( + github.com/canonical/go-dqlite/v3 v3.0.1 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/Rican7/retry v0.3.1 // indirect + github.com/google/renameio v1.0.1 // indirect + github.com/mattn/go-sqlite3 v1.14.7 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/sync v0.8.0 // indirect +) diff --git a/tools/dqlite-vacuum/go.sum b/tools/dqlite-vacuum/go.sum new file mode 100644 index 0000000..2e877e7 --- /dev/null +++ b/tools/dqlite-vacuum/go.sum @@ -0,0 +1,43 @@ +github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc= +github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0= +github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0= +github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= +github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA= +github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/dqlite-vacuum/main.go b/tools/dqlite-vacuum/main.go new file mode 100644 index 0000000..866b4f1 --- /dev/null +++ b/tools/dqlite-vacuum/main.go @@ -0,0 +1,150 @@ +// dqlite-vacuum prints page statistics of a dqlite database and, with +// -vacuum, runs VACUUM on it. +// +// The dqlite shell that ships with microk8s wraps every statement in a +// transaction, and SQLite refuses to VACUUM inside one. This sends VACUUM as +// a plain statement through the go-dqlite driver instead. dqlite supports it +// from v1.18.2 (canonical/dqlite#734). +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "database/sql" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/canonical/go-dqlite/v3/client" + "github.com/canonical/go-dqlite/v3/driver" + "gopkg.in/yaml.v2" +) + +type stats struct { + pageSize, pages, free, rows int64 +} + +func (s stats) String() string { + mb := func(pages int64) float64 { return float64(pages*s.pageSize) / 1e6 } + return fmt.Sprintf("%d pages (%.0f MB), %d free (%.0f MB), %d kine rows", + s.pages, mb(s.pages), s.free, mb(s.free), s.rows) +} + +func main() { + dir := flag.String("dir", "/var/snap/microk8s/current/var/kubernetes/backend", "dqlite backend directory with info.yaml, cluster.crt and cluster.key") + name := flag.String("db", "k8s", "database name") + vacuum := flag.Bool("vacuum", false, "run VACUUM; without it only print statistics") + timeout := flag.Duration("timeout", 10*time.Minute, "timeout for VACUUM") + flag.Parse() + log.SetFlags(log.Ltime | log.Lmicroseconds) + + var info struct { + Address string `yaml:"Address"` + } + data, err := os.ReadFile(filepath.Join(*dir, "info.yaml")) + if err != nil { + log.Fatal(err) + } + if err := yaml.Unmarshal(data, &info); err != nil || info.Address == "" { + log.Fatalf("no address in info.yaml: %v", err) + } + + dial, err := tlsDial(filepath.Join(*dir, "cluster.crt"), filepath.Join(*dir, "cluster.key")) + if err != nil { + log.Fatal(err) + } + store := client.NewInmemNodeStore() + if err := store.Set(context.Background(), []client.NodeInfo{{Address: info.Address}}); err != nil { + log.Fatal(err) + } + drv, err := driver.New(store, driver.WithDialFunc(dial), driver.WithConnectionTimeout(10*time.Second)) + if err != nil { + log.Fatal(err) + } + sql.Register("dqlite", drv) + db, err := sql.Open("dqlite", *name) + if err != nil { + log.Fatal(err) + } + defer db.Close() + // One connection, so VACUUM never runs next to a statement of our own. + db.SetMaxOpenConns(1) + + before, err := readStats(db) + if err != nil { + log.Fatal(err) + } + log.Printf("%s: %s", info.Address, before) + if !*vacuum { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + start := time.Now() + log.Print("VACUUM started") + if _, err := db.ExecContext(ctx, "VACUUM"); err != nil { + log.Fatalf("VACUUM failed after %s: %v", time.Since(start).Round(time.Millisecond), err) + } + log.Printf("VACUUM done in %s", time.Since(start).Round(time.Millisecond)) + + after, err := readStats(db) + if err != nil { + log.Fatal(err) + } + log.Printf("%s: %s", info.Address, after) +} + +func readStats(db *sql.DB) (stats, error) { + var s stats + for _, q := range []struct { + sql string + dst *int64 + }{ + {"PRAGMA page_size", &s.pageSize}, + {"PRAGMA page_count", &s.pages}, + {"PRAGMA freelist_count", &s.free}, + {"SELECT COUNT(*) FROM kine", &s.rows}, + } { + if err := db.QueryRow(q.sql).Scan(q.dst); err != nil { + return s, fmt.Errorf("%s: %w", q.sql, err) + } + } + return s, nil +} + +// tlsDial mirrors go-dqlite's app.SimpleDialTLSConfig, which can't be +// imported without cgo. +func tlsDial(crt, key string) (driver.DialFunc, error) { + cert, err := tls.LoadX509KeyPair(crt, key) + if err != nil { + return nil, err + } + pem, err := os.ReadFile(crt) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("bad certificate %s", crt) + } + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, err + } + if len(leaf.DNSNames) == 0 { + return nil, fmt.Errorf("certificate %s has no DNS name", crt) + } + config := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: pool, + Certificates: []tls.Certificate{cert}, + ClientSessionCache: tls.NewLRUClientSessionCache(0), + ServerName: leaf.DNSNames[0], + } + return client.DialFuncWithTLS(client.DefaultDialFunc, config), nil +}