feat(agent) multi-driver db monitoring, hot-plug config, release pipeline

This commit is contained in:
2026-06-15 17:29:53 +01:00
parent db6ed003f1
commit 40811ed6e8
17 changed files with 949 additions and 34 deletions

View File

@@ -11,6 +11,8 @@ import (
"bugs.watch/agent/internal/collectors/disk"
"bugs.watch/agent/internal/collectors/memory"
"bugs.watch/agent/internal/collectors/uptime"
"bugs.watch/agent/internal/config"
"bugs.watch/agent/internal/dbmanager"
"bugs.watch/agent/internal/sender"
)
@@ -19,13 +21,12 @@ const (
windowDuration = 60 * time.Second
)
func Run(ctx context.Context) {
func Run(ctx context.Context, cfg config.Config) {
log.Println("bugswatch agent starting")
log.Printf("db config: %s", cfg.DBConfigPath)
// Aggregation window
window := aggregate.NewWindow()
// Register collectors
cols := []collectors.Collector{
uptime.New(),
cpu.New(),
@@ -33,8 +34,10 @@ func Run(ctx context.Context) {
disk.New(),
}
// Sender (HTTP or logging)
snd := sender.NewHTTPSender("http://172.17.0.1:5173/api/health")
mgr := dbmanager.New(cfg.DBConfigPath)
go mgr.Watch(ctx)
snd := sender.NewHTTPSender(cfg)
ticker := time.NewTicker(sampleInterval)
defer ticker.Stop()
@@ -43,21 +46,19 @@ func Run(ctx context.Context) {
select {
case <-ctx.Done():
log.Println("bugswatch agent shutting down")
// Best-effort final flush with bounded timeout
shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
emitSnapshot(shutdownCtx, snd, window)
emitSnapshot(shutdownCtx, snd, window, mgr)
return
case <-ticker.C:
for _, c := range cols {
c.Collect(window)
}
mgr.Collect(window)
if window.Ready(windowDuration) {
emitSnapshot(ctx, snd, window)
emitSnapshot(ctx, snd, window, mgr)
window = aggregate.NewWindow()
}
}
@@ -68,6 +69,7 @@ func emitSnapshot(
ctx context.Context,
snd sender.Sender,
window *aggregate.Window,
mgr *dbmanager.Manager,
) {
snap := window.Snapshot()
_ = snd.Send(ctx, snap)

View File

@@ -15,22 +15,36 @@ type Window struct {
diskCount uint64
uptime uint64
// DB metrics — populated only when a DB collector is registered.
hasDB bool
dbDriver uint8
dbConnPctSum float64
dbConnPctCount uint64
dbThreadsSum float64
dbThreadsCount uint64
dbSlowQueriesDelta uint64
dbSizeBytes uint64
}
type Snapshot struct {
Timestamp time.Time
CPUAvg float64
MemAvg float64
CPUAvg float64
MemAvg float64
DiskAvg float64
Uptime uint64
Uptime uint64
// nil when DB monitoring is not enabled.
DBDriver *uint8
DBConnPct *float64
DBThreadsRunning *float64
DBSlowQueries *uint64
DBSizeBytes *uint64
}
func NewWindow() *Window {
return &Window{
start: time.Now(),
}
return &Window{start: time.Now()}
}
func (w *Window) AddCPU(value float64) {
@@ -43,10 +57,41 @@ func (w *Window) AddMemory(value float64) {
w.memCount++
}
func (w *Window) AddDisk(value float64) {
w.diskSum += value
w.diskCount++
}
func (w *Window) SetUptime(value uint64) {
w.uptime = value
}
func (w *Window) AddDBConnPct(v float64) {
w.dbConnPctSum += v
w.dbConnPctCount++
w.hasDB = true
}
func (w *Window) AddDBThreadsRunning(v float64) {
w.dbThreadsSum += v
w.dbThreadsCount++
}
func (w *Window) AddDBSlowQueriesDelta(v uint64) {
w.dbSlowQueriesDelta += v
}
func (w *Window) SetDBSizeBytes(v uint64) {
w.dbSizeBytes = v
w.hasDB = true
}
// SetDBDriver records the numeric driver identifier (see dbmanager.Driver* constants).
// When multiple collectors are active this is called once per window by the manager.
func (w *Window) SetDBDriver(v uint8) {
w.dbDriver = v
}
func (w *Window) Ready(windowDuration time.Duration) bool {
return time.Since(w.start) >= windowDuration
}
@@ -67,16 +112,36 @@ func (w *Window) Snapshot() Snapshot {
diskAvg = w.diskSum / float64(w.diskCount)
}
return Snapshot{
snap := Snapshot{
Timestamp: time.Now(),
CPUAvg: cpuAvg,
MemAvg: memAvg,
DiskAvg: diskAvg,
Uptime: w.uptime,
}
}
func (w *Window) AddDisk(value float64) {
w.diskSum += value
w.diskCount++
}
if w.hasDB {
driver := w.dbDriver
var connPct float64
if w.dbConnPctCount > 0 {
connPct = w.dbConnPctSum / float64(w.dbConnPctCount)
}
var threads float64
if w.dbThreadsCount > 0 {
threads = w.dbThreadsSum / float64(w.dbThreadsCount)
}
slow := w.dbSlowQueriesDelta
sizeBytes := w.dbSizeBytes
snap.DBDriver = &driver
snap.DBConnPct = &connPct
snap.DBThreadsRunning = &threads
snap.DBSlowQueries = &slow
snap.DBSizeBytes = &sizeBytes
}
return snap
}

View File

@@ -0,0 +1,94 @@
package mongodb
import (
"context"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"bugs.watch/agent/internal/aggregate"
)
// Collector queries MongoDB serverStatus and listDatabases each sample interval.
// All queries run under a 3-second timeout; any failure is silently skipped.
// Metrics mapping:
// db_conn_pct → current / (current + available) * 100
// db_threads_running → connections.current
// db_size_bytes → totalSize from listDatabases
type Collector struct {
client *mongo.Client
}
func New(uri string) (*Collector, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).
SetConnectTimeout(3*time.Second).
SetServerSelectionTimeout(3*time.Second))
if err != nil {
return nil, err
}
return &Collector{client: client}, nil
}
func (c *Collector) Name() string { return "mongodb" }
func (c *Collector) Close() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
c.client.Disconnect(ctx)
}
func (c *Collector) Collect(window *aggregate.Window) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
admin := c.client.Database("admin")
var status bson.M
if err := admin.RunCommand(ctx, bson.D{
{Key: "serverStatus", Value: 1},
{Key: "repl", Value: 0},
{Key: "metrics", Value: 0},
{Key: "locks", Value: 0},
}).Decode(&status); err != nil {
return
}
if conns, ok := status["connections"].(bson.M); ok {
current := toFloat64(conns["current"])
available := toFloat64(conns["available"])
total := current + available
if total > 0 {
window.AddDBConnPct(current / total * 100)
}
if current > 0 {
window.AddDBThreadsRunning(current)
}
}
var listResult bson.M
if err := admin.RunCommand(ctx, bson.D{
{Key: "listDatabases", Value: 1},
{Key: "nameOnly", Value: false},
}).Decode(&listResult); err == nil {
if totalSize := toFloat64(listResult["totalSize"]); totalSize > 0 {
window.SetDBSizeBytes(uint64(totalSize))
}
}
}
func toFloat64(v interface{}) float64 {
switch x := v.(type) {
case int32:
return float64(x)
case int64:
return float64(x)
case float64:
return x
}
return 0
}

View File

@@ -0,0 +1,92 @@
package mysql
import (
"context"
"database/sql"
"strconv"
"time"
_ "github.com/go-sql-driver/mysql"
"bugs.watch/agent/internal/aggregate"
)
// Collector queries MySQL global status each sample interval.
// All queries run under a 3-second timeout; any failure is silently skipped.
type Collector struct {
db *sql.DB
prevSlowQueries uint64
hasPrev bool
}
func New(dsn string) (*Collector, error) {
db, err := sql.Open("mysql", dsn)
if err != nil {
return nil, err
}
db.SetMaxOpenConns(2)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(5 * time.Minute)
return &Collector{db: db}, nil
}
func (c *Collector) Name() string { return "mysql" }
func (c *Collector) Close() { c.db.Close() }
func (c *Collector) Collect(window *aggregate.Window) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
rows, err := c.db.QueryContext(ctx,
"SHOW GLOBAL STATUS WHERE Variable_name IN ('Threads_connected','Threads_running','Slow_queries')",
)
if err != nil {
return
}
defer rows.Close()
vars := make(map[string]uint64, 3)
for rows.Next() {
var name, raw string
if err := rows.Scan(&name, &raw); err != nil {
continue
}
if v, err := strconv.ParseUint(raw, 10, 64); err == nil {
vars[name] = v
}
}
if err := rows.Err(); err != nil {
return
}
var maxConn uint64
if err := c.db.QueryRowContext(ctx, "SELECT @@max_connections").Scan(&maxConn); err != nil || maxConn == 0 {
maxConn = 1
}
connPct := float64(vars["Threads_connected"]) / float64(maxConn) * 100
window.AddDBConnPct(connPct)
window.AddDBThreadsRunning(float64(vars["Threads_running"]))
slow := vars["Slow_queries"]
if c.hasPrev {
var delta uint64
if slow >= c.prevSlowQueries {
delta = slow - c.prevSlowQueries
}
window.AddDBSlowQueriesDelta(delta)
}
c.prevSlowQueries = slow
c.hasPrev = true
var sizeBytes uint64
err = c.db.QueryRowContext(ctx,
"SELECT COALESCE(SUM(data_length + index_length), 0)"+
" FROM information_schema.tables"+
" WHERE table_schema NOT IN ('information_schema','performance_schema','mysql','sys')",
).Scan(&sizeBytes)
if err == nil {
window.SetDBSizeBytes(sizeBytes)
}
}

View File

@@ -0,0 +1,63 @@
package postgres
import (
"context"
"database/sql"
"time"
_ "github.com/lib/pq"
"bugs.watch/agent/internal/aggregate"
)
// Collector queries PostgreSQL status each sample interval.
// All queries run under a 3-second timeout; any failure is silently skipped.
type Collector struct {
db *sql.DB
}
func New(dsn string) (*Collector, error) {
db, err := sql.Open("postgres", dsn)
if err != nil {
return nil, err
}
db.SetMaxOpenConns(2)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(5 * time.Minute)
return &Collector{db: db}, nil
}
func (c *Collector) Name() string { return "postgres" }
func (c *Collector) Close() { c.db.Close() }
func (c *Collector) Collect(window *aggregate.Window) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
// pg_stat_database is readable by any user for their own database.
var numBackends, maxConn int
err := c.db.QueryRowContext(ctx,
`SELECT d.numbackends, s.setting::int
FROM pg_stat_database d, pg_settings s
WHERE d.datname = current_database() AND s.name = 'max_connections'`,
).Scan(&numBackends, &maxConn)
if err == nil && maxConn > 0 {
window.AddDBConnPct(float64(numBackends) / float64(maxConn) * 100)
}
// pg_stat_activity requires pg_monitor role (PG 10+); silently skip on error.
var activeQueries int
if err := c.db.QueryRowContext(ctx,
`SELECT count(*) FROM pg_stat_activity WHERE state = 'active'`,
).Scan(&activeQueries); err == nil {
window.AddDBThreadsRunning(float64(activeQueries))
}
var sizeBytes int64
if err := c.db.QueryRowContext(ctx,
`SELECT pg_database_size(current_database())`,
).Scan(&sizeBytes); err == nil {
window.SetDBSizeBytes(uint64(sizeBytes))
}
}

View File

@@ -0,0 +1,78 @@
package redis
import (
"context"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"bugs.watch/agent/internal/aggregate"
)
// Collector queries Redis INFO each sample interval.
// All queries run under a 3-second timeout; any failure is silently skipped.
// Metrics mapping:
// db_conn_pct → used_memory / maxmemory * 100 (skipped when maxmemory=0)
// db_threads_running → connected_clients
// db_size_bytes → used_memory
type Collector struct {
client *redis.Client
}
func New(addr string) (*Collector, error) {
opts, err := redis.ParseURL(addr)
if err != nil {
return nil, err
}
opts.DialTimeout = 3 * time.Second
opts.ReadTimeout = 3 * time.Second
opts.WriteTimeout = 3 * time.Second
return &Collector{client: redis.NewClient(opts)}, nil
}
func (c *Collector) Name() string { return "redis" }
func (c *Collector) Close() { c.client.Close() }
func (c *Collector) Collect(window *aggregate.Window) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
info, err := c.client.Info(ctx, "memory", "clients").Result()
if err != nil {
return
}
fields := parseInfo(info)
usedMem, _ := strconv.ParseUint(fields["used_memory"], 10, 64)
maxMem, _ := strconv.ParseUint(fields["maxmemory"], 10, 64)
clients, _ := strconv.ParseUint(fields["connected_clients"], 10, 64)
if usedMem > 0 {
window.SetDBSizeBytes(usedMem)
}
if clients > 0 {
window.AddDBThreadsRunning(float64(clients))
}
if maxMem > 0 && usedMem > 0 {
window.AddDBConnPct(float64(usedMem) / float64(maxMem) * 100)
}
}
func parseInfo(s string) map[string]string {
out := make(map[string]string)
for _, line := range strings.Split(s, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
k, v, ok := strings.Cut(line, ":")
if ok {
out[strings.TrimSpace(k)] = strings.TrimSpace(v)
}
}
return out
}

View File

@@ -0,0 +1,30 @@
package sqlite
import (
"os"
"bugs.watch/agent/internal/aggregate"
)
// Collector reports the on-disk size of a SQLite database file.
// No driver or connection is needed; we simply stat the file.
// Failures are silently skipped.
type Collector struct {
path string
}
func New(path string) *Collector {
return &Collector{path: path}
}
func (c *Collector) Name() string { return "sqlite" }
func (c *Collector) Close() {}
func (c *Collector) Collect(window *aggregate.Window) {
info, err := os.Stat(c.path)
if err != nil {
return
}
window.SetDBSizeBytes(uint64(info.Size()))
}

45
internal/config/config.go Normal file
View File

@@ -0,0 +1,45 @@
package config
import (
"errors"
"os"
)
const DefaultDBConfigPath = "/etc/bugswatch/databases.json"
type Config struct {
Endpoint string
AgentID string
APIToken string
HostID string
Env string
DBConfigPath string
}
func Load() (Config, error) {
dbConfig := os.Getenv("BUGSWATCH_DB_CONFIG")
if dbConfig == "" {
dbConfig = DefaultDBConfigPath
}
cfg := Config{
Endpoint: os.Getenv("BUGSWATCH_ENDPOINT"),
AgentID: os.Getenv("BUGSWATCH_AGENT_ID"),
APIToken: os.Getenv("BUGSWATCH_API_TOKEN"),
HostID: os.Getenv("BUGSWATCH_HOST_ID"),
Env: os.Getenv("BUGSWATCH_ENVIRONMENT"),
DBConfigPath: dbConfig,
}
if cfg.Endpoint == "" {
return Config{}, errors.New("BUGSWATCH_ENDPOINT not set")
}
if cfg.AgentID == "" {
return Config{}, errors.New("BUGSWATCH_AGENT_ID not set")
}
if cfg.APIToken == "" {
return Config{}, errors.New("BUGSWATCH_API_TOKEN not set")
}
return cfg, nil
}

View File

@@ -0,0 +1,203 @@
package dbmanager
import (
"context"
"crypto/md5"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sync"
"time"
"bugs.watch/agent/internal/aggregate"
"bugs.watch/agent/internal/collectors"
mongocol "bugs.watch/agent/internal/collectors/mongodb"
mysqlcol "bugs.watch/agent/internal/collectors/mysql"
pgcol "bugs.watch/agent/internal/collectors/postgres"
rediscol "bugs.watch/agent/internal/collectors/redis"
sqlitecol "bugs.watch/agent/internal/collectors/sqlite"
)
// Driver type identifiers — used as the db_driver field in the health payload
// so the dashboard can render the correct icon without parsing strings.
const (
DriverNone uint8 = 0
DriverMySQL uint8 = 1 // also MariaDB
DriverPostgres uint8 = 2
DriverRedis uint8 = 3
DriverMongoDB uint8 = 4
DriverSQLite uint8 = 5
)
// Entry is one database config record in databases.json.
type Entry struct {
Driver string `json:"driver"`
URL string `json:"url"`
}
type dbCollector interface {
collectors.Collector
Close()
}
// Manager watches a config file and hot-swaps DB collectors without agent restart.
// Collect() is safe to call concurrently with Watch().
type Manager struct {
mu sync.RWMutex
active []dbCollector
configPath string
lastHash [16]byte
}
func New(configPath string) *Manager {
return &Manager{configPath: configPath}
}
// Watch polls the config file every 30 seconds and reloads on change.
// Blocks until ctx is cancelled.
func (m *Manager) Watch(ctx context.Context) {
m.reload()
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
m.swap(nil)
return
case <-ticker.C:
m.reload()
}
}
}
// Collect runs all active DB collectors against the window.
// Holds RLock for the full duration so swaps never race with in-progress queries.
func (m *Manager) Collect(window *aggregate.Window) {
m.mu.RLock()
defer m.mu.RUnlock()
if len(m.active) == 0 {
return
}
window.SetDBDriver(m.driverID(m.active[0].Name()))
for _, c := range m.active {
c.Collect(window)
}
}
func (m *Manager) driverID(name string) uint8 {
switch name {
case "mysql", "mariadb":
return DriverMySQL
case "postgres":
return DriverPostgres
case "redis":
return DriverRedis
case "mongodb":
return DriverMongoDB
case "sqlite":
return DriverSQLite
default:
return DriverNone
}
}
func (m *Manager) reload() {
data, err := os.ReadFile(m.configPath)
if err != nil {
if os.IsNotExist(err) {
m.swap(nil)
}
return
}
hash := md5.Sum(data)
m.mu.RLock()
unchanged := hash == m.lastHash
m.mu.RUnlock()
if unchanged {
return
}
var entries []Entry
if err := json.Unmarshal(data, &entries); err != nil {
log.Printf("dbmanager: invalid config: %v", err)
return
}
var next []dbCollector
for _, e := range entries {
c, err := newCollector(e)
if err != nil {
log.Printf("dbmanager: %s: %v", e.Driver, err)
continue
}
next = append(next, c)
log.Printf("dbmanager: loaded %s", e.Driver)
}
m.swap(next)
m.mu.Lock()
m.lastHash = hash
m.mu.Unlock()
}
func (m *Manager) swap(next []dbCollector) {
m.mu.Lock()
old := m.active
m.active = next
m.mu.Unlock()
for _, c := range old {
c.Close()
}
}
func newCollector(e Entry) (dbCollector, error) {
switch e.Driver {
case "mysql", "mariadb":
return mysqlcol.New(e.URL)
case "postgres", "postgresql":
return pgcol.New(e.URL)
case "redis":
return rediscol.New(e.URL)
case "mongodb":
return mongocol.New(e.URL)
case "sqlite":
return sqlitecol.New(e.URL), nil
default:
return nil, fmt.Errorf("unknown driver %q (supported: mysql, mariadb, postgres, redis, mongodb, sqlite)", e.Driver)
}
}
// ReadEntries reads the config file, returning an empty slice if it doesn't exist.
func ReadEntries(configPath string) ([]Entry, error) {
data, err := os.ReadFile(configPath)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, err
}
var entries []Entry
if err := json.Unmarshal(data, &entries); err != nil {
return nil, err
}
return entries, nil
}
// WriteEntries writes entries to the config file, creating the directory if needed.
func WriteEntries(configPath string, entries []Entry) error {
if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil {
return err
}
data, err := json.MarshalIndent(entries, "", " ")
if err != nil {
return err
}
return os.WriteFile(configPath, data, 0644)
}

View File

@@ -7,30 +7,44 @@ import (
"net/http"
"time"
"bugs.watch/agent/internal/config"
"bugs.watch/agent/internal/aggregate"
)
type HTTPSender struct {
client *http.Client
endpoint string
token string
hostID string
env string
}
func NewHTTPSender(endpoint string) *HTTPSender {
func NewHTTPSender(cfg config.Config) *HTTPSender {
return &HTTPSender{
client: &http.Client{
Timeout: 5 * time.Second,
},
endpoint: endpoint,
endpoint: cfg.Endpoint + "/api/v1/agents/" + cfg.AgentID,
token: cfg.APIToken,
hostID: cfg.HostID,
env: cfg.Env,
}
}
func (s *HTTPSender) Send(ctx context.Context, snap aggregate.Snapshot) error {
payload := SnapshotPayload{
Timestamp: snap.Timestamp,
CPUAvg: snap.CPUAvg,
MemAvg: snap.MemAvg,
DiskAvg: snap.DiskAvg,
Uptime: snap.Uptime,
Timestamp: snap.Timestamp,
CPUAvg: snap.CPUAvg,
MemAvg: snap.MemAvg,
DiskAvg: snap.DiskAvg,
Uptime: snap.Uptime,
HostID: s.hostID,
Environment: s.env,
DBDriver: snap.DBDriver,
DBConnPct: snap.DBConnPct,
DBThreadsRunning: snap.DBThreadsRunning,
DBSlowQueries: snap.DBSlowQueries,
DBSizeBytes: snap.DBSizeBytes,
}
body, err := json.Marshal(payload)
@@ -40,7 +54,7 @@ func (s *HTTPSender) Send(ctx context.Context, snap aggregate.Snapshot) error {
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
http.MethodPut,
s.endpoint,
bytes.NewReader(body),
)
@@ -49,6 +63,7 @@ func (s *HTTPSender) Send(ctx context.Context, snap aggregate.Snapshot) error {
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer " + s.token)
resp, err := s.client.Do(req)
if err != nil {

View File

@@ -9,6 +9,16 @@ type SnapshotPayload struct {
CPUAvg float64 `json:"cpu_avg"`
MemAvg float64 `json:"mem_avg"`
DiskAvg float64 `json:"disk_avg"`
Uptime uint64 `json:"uptime"`
Uptime uint64 `json:"uptime"`
HostID string `json:"host_id,omitempty"`
Environment string `json:"environment,omitempty"`
// Optional — only present when DB monitoring is enabled.
// DBDriver is a numeric identifier: 1=mysql/mariadb 2=postgres 3=redis 4=mongodb 5=sqlite
DBDriver *uint8 `json:"db_driver,omitempty"`
DBConnPct *float64 `json:"db_conn_pct,omitempty"`
DBThreadsRunning *float64 `json:"db_threads_running,omitempty"`
DBSlowQueries *uint64 `json:"db_slow_queries,omitempty"`
DBSizeBytes *uint64 `json:"db_size_bytes,omitempty"`
}