mirror of
https://github.com/rubenhensen/k8scd.git
synced 2026-11-15 14:29:09 +01:00
Add a tool to VACUUM the k8s-dqlite database
dqlite supports VACUUM from v1.18.2, which microk8s 1.34 ships, but the dqlite shell wraps every statement in a transaction and SQLite refuses to VACUUM inside one. dqlite-vacuum sends it through the go-dqlite driver instead and otherwise just prints page statistics. dqlite-vacuum.sh makes the backups first. Rehearsed on opti1 on a throwaway node in /dev/shm: 657 MB to 41 MB in 0.6s, with concurrent writes unaffected and the data intact after restarts.
This commit is contained in:
Ruben Hensen committed
1 parent
c55b7ceab5
commit
175769d36c
5 files changed
+276
-4
No files matched your search
@@ -230,8 +230,31 @@ Findings from 2026-10-23:
|
||||
- [ ] `sudo snap refresh microk8s --channel=1.36/stable`, addons repo, Calico
|
||||
|
||||
### Shrink the database
|
||||
- [ ] On dqlite 1.18.2+ (so from microk8s 1.34): offline backup, then a
|
||||
supervised VACUUM (the maintainers advise against running it
|
||||
unattended). Expect the file to go from ~709 MB to ~35 MB and snapshots
|
||||
to shrink with it
|
||||
|
||||
dqlite accepts VACUUM since v1.18.2 (canonical/dqlite#734). Its maintainer:
|
||||
"it potentially create a very big transaction which has effects both on the
|
||||
RAFT protocol *and* on resource usage. I would recommend not to run it
|
||||
unsupervised." The dqlite shell that ships with microk8s can't send it: it
|
||||
wraps every statement in a transaction, and SQLite refuses to VACUUM inside
|
||||
one. `tools/dqlite-vacuum` sends it through the go-dqlite driver as a plain
|
||||
statement, as user `ruben` (in the `microk8s` group, which can read
|
||||
`cluster.crt`, `cluster.key` and `info.yaml`); without `-vacuum` it only
|
||||
prints page statistics.
|
||||
|
||||
Rehearsed on 2026-10-23 on opti1 with the k8s-dqlite and dqlite binaries
|
||||
of microk8s 1.34, on a throwaway node in `/dev/shm` (not in Docker on the
|
||||
Mac: the amd64 emulation lacks `io_setup`, which raft needs): 657 MB with
|
||||
616 MB free went to 41 MB with none free in 0.6s; a writer next to it saw
|
||||
no failures and no write over 1s; after a restart, and after a restart from
|
||||
the first snapshot (40 MB), `integrity_check` was ok and all rows were
|
||||
there.
|
||||
|
||||
Live database before: 709 MB, 666 MB free, ~4,400 kine rows; snapshots of
|
||||
~150 MB every ~23 minutes; k8s-dqlite RSS 1,346 MB.
|
||||
|
||||
- [ ] `ssh -t ruben@192.168.1.204 ./dqlite-vacuum.sh`: dbctl backup, an
|
||||
offline copy of the backend (API down ~1 min), VACUUM, integrity check.
|
||||
How to go back is at the top of the script
|
||||
- [ ] Afterwards: the next snapshot's size (expect ~8 MB instead of
|
||||
~150 MB), lease update times, controller restarts, k8s-dqlite memory
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# VACUUM the k8s-dqlite database, see "Shrink the database" in
|
||||
# docs/upgrade-k8s-1.33.md. Copy this script and the tool next to it to the
|
||||
# home directory on opti1; build the tool with
|
||||
# CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dqlite-vacuum .
|
||||
#
|
||||
# Rehearsed on a throwaway node with the binaries of microk8s 1.34: 657 MB to
|
||||
# 41 MB in 0.6s, writes next to it kept working, and the data was intact
|
||||
# after a restart and after the first snapshot.
|
||||
#
|
||||
# To go back to the state before the VACUUM:
|
||||
# sudo snap stop microk8s.daemon-k8s-dqlite
|
||||
# cd /var/snap/microk8s/current/var/kubernetes
|
||||
# sudo mv backend backend.vacuumed && sudo tar -xpf /root/backend-pre-vacuum-<stamp>.tar
|
||||
# sudo snap start microk8s.daemon-k8s-dqlite
|
||||
set -euo pipefail
|
||||
stamp=$(date +%Y%m%d-%H%M)
|
||||
kube=/var/snap/microk8s/current/var/kubernetes
|
||||
shell="/snap/microk8s/current/bin/dqlite -s 127.0.0.1:19001 -c $kube/backend/cluster.crt -k $kube/backend/cluster.key --timeout 60000 k8s"
|
||||
echo "== $(date) start"
|
||||
|
||||
echo "== datastore backup to /root/dqlite-pre-vacuum-$stamp.tar.gz"
|
||||
sudo microk8s dbctl backup -o /root/dqlite-pre-vacuum-$stamp
|
||||
|
||||
echo "== offline copy of the backend (API down ~1 min, pods keep running)"
|
||||
sudo snap stop microk8s.daemon-k8s-dqlite
|
||||
sudo tar -cpf /root/backend-pre-vacuum-$stamp.tar -C $kube backend
|
||||
sudo snap start microk8s.daemon-k8s-dqlite
|
||||
sudo ls -la /root/backend-pre-vacuum-$stamp.tar /root/dqlite-pre-vacuum-$stamp.tar.gz
|
||||
sudo microk8s status --wait-ready --timeout 300 >/dev/null && echo "microk8s is ready"
|
||||
echo "== letting the API server and controllers settle for a minute"
|
||||
sleep 60
|
||||
|
||||
echo "== VACUUM"
|
||||
ps -o rss= -C k8s-dqlite | awk '{printf "k8s-dqlite RSS before: %.0f MB\n", $1/1024}'
|
||||
~/dqlite-vacuum -vacuum
|
||||
echo "== integrity check"
|
||||
$shell "PRAGMA integrity_check"
|
||||
ps -o rss= -C k8s-dqlite | awk '{printf "k8s-dqlite RSS after: %.0f MB\n", $1/1024}'
|
||||
echo "== $(date) done"
|
||||
@@ -0,0 +1,16 @@
|
||||
module dqlite-vacuum
|
||||
|
||||
go 1.27.3
|
||||
|
||||
require (
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Rican7/retry v0.3.1 // indirect
|
||||
github.com/google/renameio v1.0.1 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.7 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
golang.org/x/sync v0.8.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc=
|
||||
github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1 h1:eUTcEHetVi1+zr+ZFz7PU0G+0eK66GZ7DKhi+wHs1f0=
|
||||
github.com/canonical/go-dqlite/v3 v3.0.1/go.mod h1:Kb/9JVog9XeIlnlnSyO1JgMoE4PoKXZCB8fJNcvvHAc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU=
|
||||
github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
|
||||
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,150 @@
|
||||
// dqlite-vacuum prints page statistics of a dqlite database and, with
|
||||
// -vacuum, runs VACUUM on it.
|
||||
//
|
||||
// The dqlite shell that ships with microk8s wraps every statement in a
|
||||
// transaction, and SQLite refuses to VACUUM inside one. This sends VACUUM as
|
||||
// a plain statement through the go-dqlite driver instead. dqlite supports it
|
||||
// from v1.18.2 (canonical/dqlite#734).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/canonical/go-dqlite/v3/client"
|
||||
"github.com/canonical/go-dqlite/v3/driver"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type stats struct {
|
||||
pageSize, pages, free, rows int64
|
||||
}
|
||||
|
||||
func (s stats) String() string {
|
||||
mb := func(pages int64) float64 { return float64(pages*s.pageSize) / 1e6 }
|
||||
return fmt.Sprintf("%d pages (%.0f MB), %d free (%.0f MB), %d kine rows",
|
||||
s.pages, mb(s.pages), s.free, mb(s.free), s.rows)
|
||||
}
|
||||
|
||||
func main() {
|
||||
dir := flag.String("dir", "/var/snap/microk8s/current/var/kubernetes/backend", "dqlite backend directory with info.yaml, cluster.crt and cluster.key")
|
||||
name := flag.String("db", "k8s", "database name")
|
||||
vacuum := flag.Bool("vacuum", false, "run VACUUM; without it only print statistics")
|
||||
timeout := flag.Duration("timeout", 10*time.Minute, "timeout for VACUUM")
|
||||
flag.Parse()
|
||||
log.SetFlags(log.Ltime | log.Lmicroseconds)
|
||||
|
||||
var info struct {
|
||||
Address string `yaml:"Address"`
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(*dir, "info.yaml"))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &info); err != nil || info.Address == "" {
|
||||
log.Fatalf("no address in info.yaml: %v", err)
|
||||
}
|
||||
|
||||
dial, err := tlsDial(filepath.Join(*dir, "cluster.crt"), filepath.Join(*dir, "cluster.key"))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
store := client.NewInmemNodeStore()
|
||||
if err := store.Set(context.Background(), []client.NodeInfo{{Address: info.Address}}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
drv, err := driver.New(store, driver.WithDialFunc(dial), driver.WithConnectionTimeout(10*time.Second))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
sql.Register("dqlite", drv)
|
||||
db, err := sql.Open("dqlite", *name)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
// One connection, so VACUUM never runs next to a statement of our own.
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
before, err := readStats(db)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("%s: %s", info.Address, before)
|
||||
if !*vacuum {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
log.Print("VACUUM started")
|
||||
if _, err := db.ExecContext(ctx, "VACUUM"); err != nil {
|
||||
log.Fatalf("VACUUM failed after %s: %v", time.Since(start).Round(time.Millisecond), err)
|
||||
}
|
||||
log.Printf("VACUUM done in %s", time.Since(start).Round(time.Millisecond))
|
||||
|
||||
after, err := readStats(db)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("%s: %s", info.Address, after)
|
||||
}
|
||||
|
||||
func readStats(db *sql.DB) (stats, error) {
|
||||
var s stats
|
||||
for _, q := range []struct {
|
||||
sql string
|
||||
dst *int64
|
||||
}{
|
||||
{"PRAGMA page_size", &s.pageSize},
|
||||
{"PRAGMA page_count", &s.pages},
|
||||
{"PRAGMA freelist_count", &s.free},
|
||||
{"SELECT COUNT(*) FROM kine", &s.rows},
|
||||
} {
|
||||
if err := db.QueryRow(q.sql).Scan(q.dst); err != nil {
|
||||
return s, fmt.Errorf("%s: %w", q.sql, err)
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// tlsDial mirrors go-dqlite's app.SimpleDialTLSConfig, which can't be
|
||||
// imported without cgo.
|
||||
func tlsDial(crt, key string) (driver.DialFunc, error) {
|
||||
cert, err := tls.LoadX509KeyPair(crt, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pem, err := os.ReadFile(crt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, fmt.Errorf("bad certificate %s", crt)
|
||||
}
|
||||
leaf, err := x509.ParseCertificate(cert.Certificate[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(leaf.DNSNames) == 0 {
|
||||
return nil, fmt.Errorf("certificate %s has no DNS name", crt)
|
||||
}
|
||||
config := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
RootCAs: pool,
|
||||
Certificates: []tls.Certificate{cert},
|
||||
ClientSessionCache: tls.NewLRUClientSessionCache(0),
|
||||
ServerName: leaf.DNSNames[0],
|
||||
}
|
||||
return client.DialFuncWithTLS(client.DefaultDialFunc, config), nil
|
||||
}
|
||||
Reference in new issue
Block a user