Add BBolt database backend for HDD-optimized archival relays (v0.48.0)

- BBolt B+tree backend with sequential access patterns for spinning disks
- Write batching (5000 events / 128MB / 30s flush) to reduce disk thrashing
- Adjacency list storage for graph data (one key per vertex, not per edge)
- Bloom filter for fast negative edge existence checks (~12MB for 10M edges)
- No query cache (saves RAM, B+tree reads are fast enough on HDD)
- Migration tool: orly migrate --from badger --to bbolt
- Configuration: ORLY_BBOLT_* environment variables

Files modified:
- app/config/config.go: Added BBolt configuration options
- main.go: Added migrate subcommand and BBolt config wiring
- pkg/database/factory.go: Added BBolt factory registration
- pkg/bbolt/*: New BBolt database backend implementation

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
woikos
2026-01-06 06:50:39 +01:00
parent 8dfd25613d
commit 9fed1261ad
21 changed files with 3118 additions and 5 deletions

View File

@@ -95,8 +95,16 @@ type C struct {
NIP43InviteExpiry time.Duration `env:"ORLY_NIP43_INVITE_EXPIRY" default:"24h" usage:"how long invite codes remain valid"`
// Database configuration
DBType string `env:"ORLY_DB_TYPE" default:"badger" usage:"database backend to use: badger or neo4j"`
DBType string `env:"ORLY_DB_TYPE" default:"badger" usage:"database backend to use: badger, bbolt, or neo4j"`
QueryCacheDisabled bool `env:"ORLY_QUERY_CACHE_DISABLED" default:"true" usage:"disable query cache to reduce memory usage (trades memory for query performance)"`
// BBolt configuration (only used when ORLY_DB_TYPE=bbolt)
BboltBatchMaxEvents int `env:"ORLY_BBOLT_BATCH_MAX_EVENTS" default:"5000" usage:"max events before flush (tuned for HDD, only used when ORLY_DB_TYPE=bbolt)"`
BboltBatchMaxMB int `env:"ORLY_BBOLT_BATCH_MAX_MB" default:"128" usage:"max batch size in MB before flush (only used when ORLY_DB_TYPE=bbolt)"`
BboltFlushTimeout int `env:"ORLY_BBOLT_FLUSH_TIMEOUT_SEC" default:"30" usage:"max seconds before flush (only used when ORLY_DB_TYPE=bbolt)"`
BboltBloomSizeMB int `env:"ORLY_BBOLT_BLOOM_SIZE_MB" default:"16" usage:"bloom filter size in MB for edge queries (only used when ORLY_DB_TYPE=bbolt)"`
BboltNoSync bool `env:"ORLY_BBOLT_NO_SYNC" default:"false" usage:"disable fsync for performance (DANGEROUS - data loss risk, only used when ORLY_DB_TYPE=bbolt)"`
BboltMmapSizeMB int `env:"ORLY_BBOLT_MMAP_SIZE_MB" default:"8192" usage:"initial mmap size in MB (only used when ORLY_DB_TYPE=bbolt)"`
QueryCacheSizeMB int `env:"ORLY_QUERY_CACHE_SIZE_MB" default:"512" usage:"query cache size in MB (caches database query results for faster REQ responses)"`
QueryCacheMaxAge string `env:"ORLY_QUERY_CACHE_MAX_AGE" default:"5m" usage:"maximum age for cached query results (e.g., 5m, 10m, 1h)"`
@@ -357,6 +365,45 @@ func CuratingModeRequested() (requested bool, ownerKey string) {
return
}
// MigrateRequested checks if the first command line argument is "migrate"
// and returns the migration parameters.
//
// Return Values
// - requested: true if the 'migrate' subcommand was provided
// - fromType: source database type (badger, bbolt, neo4j)
// - toType: destination database type
// - targetPath: optional target path for destination database
func MigrateRequested() (requested bool, fromType, toType, targetPath string) {
if len(os.Args) > 1 {
switch strings.ToLower(os.Args[1]) {
case "migrate":
requested = true
// Parse --from, --to, --target-path flags
for i := 2; i < len(os.Args); i++ {
arg := os.Args[i]
switch {
case strings.HasPrefix(arg, "--from="):
fromType = strings.TrimPrefix(arg, "--from=")
case strings.HasPrefix(arg, "--to="):
toType = strings.TrimPrefix(arg, "--to=")
case strings.HasPrefix(arg, "--target-path="):
targetPath = strings.TrimPrefix(arg, "--target-path=")
case arg == "--from" && i+1 < len(os.Args):
i++
fromType = os.Args[i]
case arg == "--to" && i+1 < len(os.Args):
i++
toType = os.Args[i]
case arg == "--target-path" && i+1 < len(os.Args):
i++
targetPath = os.Args[i]
}
}
}
}
return
}
// KV is a key/value pair.
type KV struct{ Key, Value string }
@@ -488,18 +535,20 @@ func PrintHelp(cfg *C, printer io.Writer) {
)
_, _ = fmt.Fprintf(
printer,
`Usage: %s [env|help|identity|serve|version]
`Usage: %s [env|help|identity|migrate|serve|version]
- env: print environment variables configuring %s
- help: print this help text
- identity: print the relay identity secret and public key
- migrate: migrate data between database backends
Example: %s migrate --from badger --to bbolt
- serve: start ephemeral relay with RAM-based storage at /dev/shm/orlyserve
listening on 0.0.0.0:10547 with 'none' ACL mode (open relay)
useful for testing and benchmarking
- version: print version and exit (also: -v, --v, -version, --version)
`,
cfg.AppName, cfg.AppName,
cfg.AppName, cfg.AppName, cfg.AppName,
)
_, _ = fmt.Fprintf(
printer,
@@ -707,3 +756,22 @@ func (cfg *C) GetGraphConfigValues() (
cfg.GraphMaxResults,
cfg.GraphRateLimitRPM
}
// GetBboltConfigValues returns the BBolt database configuration values.
// This avoids circular imports with pkg/bbolt while allowing main.go to construct
// the BBolt-specific configuration.
func (cfg *C) GetBboltConfigValues() (
batchMaxEvents int,
batchMaxBytes int64,
flushTimeoutSec int,
bloomSizeMB int,
noSync bool,
mmapSizeBytes int,
) {
return cfg.BboltBatchMaxEvents,
int64(cfg.BboltBatchMaxMB) * 1024 * 1024,
cfg.BboltFlushTimeout,
cfg.BboltBloomSizeMB,
cfg.BboltNoSync,
cfg.BboltMmapSizeMB * 1024 * 1024
}