Add BBolt database backend for HDD-optimized archival relays (v0.48.0)

- BBolt B+tree backend with sequential access patterns for spinning disks
- Write batching (5000 events / 128MB / 30s flush) to reduce disk thrashing
- Adjacency list storage for graph data (one key per vertex, not per edge)
- Bloom filter for fast negative edge existence checks (~12MB for 10M edges)
- No query cache (saves RAM, B+tree reads are fast enough on HDD)
- Migration tool: orly migrate --from badger --to bbolt
- Configuration: ORLY_BBOLT_* environment variables

Files modified:
- app/config/config.go: Added BBolt configuration options
- main.go: Added migrate subcommand and BBolt config wiring
- pkg/database/factory.go: Added BBolt factory registration
- pkg/bbolt/*: New BBolt database backend implementation

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
woikos
2026-01-06 06:50:39 +01:00
parent 8dfd25613d
commit 9fed1261ad
21 changed files with 3118 additions and 5 deletions

View File

@@ -41,6 +41,14 @@ type DatabaseConfig struct {
Neo4jFetchSize int // ORLY_NEO4J_FETCH_SIZE - max records per fetch batch (default: 1000)
Neo4jMaxTxRetrySeconds int // ORLY_NEO4J_MAX_TX_RETRY_SEC - max transaction retry time (default: 30)
Neo4jQueryResultLimit int // ORLY_NEO4J_QUERY_RESULT_LIMIT - max results per query (default: 10000, 0=unlimited)
// BBolt-specific settings (optimized for HDD)
BboltBatchMaxEvents int // ORLY_BBOLT_BATCH_MAX_EVENTS - max events per batch (default: 5000)
BboltBatchMaxBytes int64 // ORLY_BBOLT_BATCH_MAX_MB * 1024 * 1024 (default: 128MB)
BboltFlushTimeout time.Duration // ORLY_BBOLT_BATCH_FLUSH_SEC * time.Second (default: 30s)
BboltBloomSizeMB int // ORLY_BBOLT_BLOOM_SIZE_MB - bloom filter size (default: 16MB)
BboltNoSync bool // ORLY_BBOLT_NO_SYNC - disable fsync (DANGEROUS)
BboltMmapSize int // ORLY_BBOLT_MMAP_SIZE_GB * 1024 * 1024 * 1024 (default: 8GB)
}
// NewDatabase creates a database instance based on the specified type.
@@ -84,8 +92,14 @@ func NewDatabaseWithConfig(
return nil, fmt.Errorf("wasmdb database backend not available (import _ \"next.orly.dev/pkg/wasmdb\")")
}
return newWasmDBDatabase(ctx, cancel, cfg)
case "bbolt", "bolt":
// Use the bbolt implementation (B+tree, optimized for HDD)
if newBboltDatabase == nil {
return nil, fmt.Errorf("bbolt database backend not available (import _ \"next.orly.dev/pkg/bbolt\")")
}
return newBboltDatabase(ctx, cancel, cfg)
default:
return nil, fmt.Errorf("unsupported database type: %s (supported: badger, neo4j, wasmdb)", dbType)
return nil, fmt.Errorf("unsupported database type: %s (supported: badger, neo4j, wasmdb, bbolt)", dbType)
}
}
@@ -108,3 +122,13 @@ var newWasmDBDatabase func(context.Context, context.CancelFunc, *DatabaseConfig)
func RegisterWasmDBFactory(factory func(context.Context, context.CancelFunc, *DatabaseConfig) (Database, error)) {
newWasmDBDatabase = factory
}
// newBboltDatabase creates a bbolt database instance
// This is defined here to avoid import cycles
var newBboltDatabase func(context.Context, context.CancelFunc, *DatabaseConfig) (Database, error)
// RegisterBboltFactory registers the bbolt database factory
// This is called from the bbolt package's init() function
func RegisterBboltFactory(factory func(context.Context, context.CancelFunc, *DatabaseConfig) (Database, error)) {
newBboltDatabase = factory
}