From 290fcbf8f0aaaf56eafcf1505972c674131d40dc Mon Sep 17 00:00:00 2001 From: mleku Date: Wed, 3 Dec 2025 21:24:43 +0000 Subject: [PATCH] remove outdated configuration items for obsolete tail packing optimization --- .claude/settings.local.json | 3 +- CLAUDE.md | 1 - app/config/config.go | 3 - main.go | 26 +++--- ...torage_test.go => compact-storage_test.go} | 85 +++++++------------ pkg/database/database.go | 45 +++++----- pkg/database/factory.go | 9 +- pkg/database/factory_wasm.go | 9 +- pkg/version/version | 2 +- 9 files changed, 74 insertions(+), 109 deletions(-) rename pkg/database/{inline-storage_test.go => compact-storage_test.go} (86%) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 8fb11af..c759282 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -179,7 +179,8 @@ "Bash(CGO_ENABLED=0 LOG_LEVEL=trace go test:*)", "Bash(go vet:*)", "Bash(gofmt:*)", - "Skill(cypher)" + "Skill(cypher)", + "Bash(git mv:*)" ], "deny": [], "ask": [] diff --git a/CLAUDE.md b/CLAUDE.md index aafbf3e..82dd7bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,7 +156,6 @@ export ORLY_QUERY_CACHE_MAX_AGE=5m # Cache expiry time export ORLY_DB_BLOCK_CACHE_MB=512 # Block cache size export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size export ORLY_DB_ZSTD_LEVEL=1 # ZSTD level: 0=off, 1=fast, 3=default, 9=best -export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes) # Serial cache for compact event storage (Badger backend) export ORLY_SERIAL_CACHE_PUBKEYS=100000 # Max pubkeys to cache (~3.2MB memory) diff --git a/app/config/config.go b/app/config/config.go index 45d59bd..2b29d05 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -101,7 +101,6 @@ type C struct { Neo4jPassword string `env:"ORLY_NEO4J_PASSWORD" default:"password" usage:"Neo4j authentication password (only used when ORLY_DB_TYPE=neo4j)"` // Advanced database tuning - InlineEventThreshold int `env:"ORLY_INLINE_EVENT_THRESHOLD" default:"1024" usage:"size threshold in bytes for inline event storage in Badger (0 to disable, typical values: 384-1024)"` SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"` SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"` @@ -411,7 +410,6 @@ func (cfg *C) GetDatabaseConfigValues() ( dataDir, logLevel string, blockCacheMB, indexCacheMB, queryCacheSizeMB int, queryCacheMaxAge time.Duration, - inlineEventThreshold int, serialCachePubkeys, serialCacheEventIds int, zstdLevel int, neo4jURI, neo4jUser, neo4jPassword string, @@ -427,7 +425,6 @@ func (cfg *C) GetDatabaseConfigValues() ( return cfg.DataDir, cfg.DBLogLevel, cfg.DBBlockCacheMB, cfg.DBIndexCacheMB, cfg.QueryCacheSizeMB, queryCacheMaxAge, - cfg.InlineEventThreshold, cfg.SerialCachePubkeys, cfg.SerialCacheEventIds, cfg.DBZSTDLevel, cfg.Neo4jURI, cfg.Neo4jUser, cfg.Neo4jPassword diff --git a/main.go b/main.go index 66670b7..4c7e5e0 100644 --- a/main.go +++ b/main.go @@ -444,24 +444,22 @@ func makeDatabaseConfig(cfg *config.C) *database.DatabaseConfig { dataDir, logLevel, blockCacheMB, indexCacheMB, queryCacheSizeMB, queryCacheMaxAge, - inlineEventThreshold, serialCachePubkeys, serialCacheEventIds, zstdLevel, neo4jURI, neo4jUser, neo4jPassword := cfg.GetDatabaseConfigValues() return &database.DatabaseConfig{ - DataDir: dataDir, - LogLevel: logLevel, - BlockCacheMB: blockCacheMB, - IndexCacheMB: indexCacheMB, - QueryCacheSizeMB: queryCacheSizeMB, - QueryCacheMaxAge: queryCacheMaxAge, - InlineEventThreshold: inlineEventThreshold, - SerialCachePubkeys: serialCachePubkeys, - SerialCacheEventIds: serialCacheEventIds, - ZSTDLevel: zstdLevel, - Neo4jURI: neo4jURI, - Neo4jUser: neo4jUser, - Neo4jPassword: neo4jPassword, + DataDir: dataDir, + LogLevel: logLevel, + BlockCacheMB: blockCacheMB, + IndexCacheMB: indexCacheMB, + QueryCacheSizeMB: queryCacheSizeMB, + QueryCacheMaxAge: queryCacheMaxAge, + SerialCachePubkeys: serialCachePubkeys, + SerialCacheEventIds: serialCacheEventIds, + ZSTDLevel: zstdLevel, + Neo4jURI: neo4jURI, + Neo4jUser: neo4jUser, + Neo4jPassword: neo4jPassword, } } diff --git a/pkg/database/inline-storage_test.go b/pkg/database/compact-storage_test.go similarity index 86% rename from pkg/database/inline-storage_test.go rename to pkg/database/compact-storage_test.go index fc06f88..98ab099 100644 --- a/pkg/database/inline-storage_test.go +++ b/pkg/database/compact-storage_test.go @@ -19,11 +19,12 @@ import ( "git.mleku.dev/mleku/nostr/interfaces/signer/p8k" ) -// TestInlineSmallEventStorage tests the Reiser4-inspired inline storage optimization -// for small events (<=1024 bytes by default). -func TestInlineSmallEventStorage(t *testing.T) { +// TestCompactEventStorage tests the compact storage format (cmp prefix) which +// replaced the old inline storage optimization (sev/evt prefixes). +// All events are now stored in compact format regardless of size. +func TestCompactEventStorage(t *testing.T) { // Create a temporary directory for the database - tempDir, err := os.MkdirTemp("", "test-inline-db-*") + tempDir, err := os.MkdirTemp("", "test-compact-db-*") if err != nil { t.Fatalf("Failed to create temporary directory: %v", err) } @@ -46,8 +47,8 @@ func TestInlineSmallEventStorage(t *testing.T) { t.Fatal(err) } - // Test Case 1: Small event (should use inline storage) - t.Run("SmallEventInlineStorage", func(t *testing.T) { + // Test Case 1: Small event (should use compact storage) + t.Run("SmallEventCompactStorage", func(t *testing.T) { smallEvent := event.New() smallEvent.Kind = kind.TextNote.K smallEvent.CreatedAt = timestamp.Now().V @@ -65,49 +66,27 @@ func TestInlineSmallEventStorage(t *testing.T) { t.Fatalf("Failed to save small event: %v", err) } - // Verify it was stored with sev prefix + // Verify it was stored with cmp prefix serial, err := db.GetSerialById(smallEvent.ID) if err != nil { t.Fatalf("Failed to get serial for small event: %v", err) } - // Check that sev key exists - sevKeyExists := false + // Check that cmp key exists (compact format) + cmpKeyExists := false db.View(func(txn *badger.Txn) error { - smallBuf := new(bytes.Buffer) - indexes.SmallEventEnc(serial).MarshalWrite(smallBuf) + cmpBuf := new(bytes.Buffer) + indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf) - opts := badger.DefaultIteratorOptions - opts.Prefix = smallBuf.Bytes() - it := txn.NewIterator(opts) - defer it.Close() - - it.Rewind() - if it.Valid() { - sevKeyExists = true - } - return nil - }) - - if !sevKeyExists { - t.Errorf("Small event was not stored with sev prefix") - } - - // Verify evt key does NOT exist for small event - evtKeyExists := false - db.View(func(txn *badger.Txn) error { - buf := new(bytes.Buffer) - indexes.EventEnc(serial).MarshalWrite(buf) - - _, err := txn.Get(buf.Bytes()) + _, err := txn.Get(cmpBuf.Bytes()) if err == nil { - evtKeyExists = true + cmpKeyExists = true } return nil }) - if evtKeyExists { - t.Errorf("Small event should not have evt key (should only use sev)") + if !cmpKeyExists { + t.Errorf("Small event was not stored with cmp prefix (compact format)") } // Fetch and verify the event @@ -124,12 +103,12 @@ func TestInlineSmallEventStorage(t *testing.T) { } }) - // Test Case 2: Large event (should use traditional storage) - t.Run("LargeEventTraditionalStorage", func(t *testing.T) { + // Test Case 2: Large event (should also use compact storage) + t.Run("LargeEventCompactStorage", func(t *testing.T) { largeEvent := event.New() largeEvent.Kind = kind.TextNote.K largeEvent.CreatedAt = timestamp.Now().V - // Create content larger than 1024 bytes (the default inline storage threshold) + // Create larger content largeContent := make([]byte, 1500) for i := range largeContent { largeContent[i] = 'x' @@ -148,27 +127,27 @@ func TestInlineSmallEventStorage(t *testing.T) { t.Fatalf("Failed to save large event: %v", err) } - // Verify it was stored with evt prefix + // Verify it was stored with cmp prefix (compact format) serial, err := db.GetSerialById(largeEvent.ID) if err != nil { t.Fatalf("Failed to get serial for large event: %v", err) } - // Check that evt key exists - evtKeyExists := false + // Check that cmp key exists + cmpKeyExists := false db.View(func(txn *badger.Txn) error { - buf := new(bytes.Buffer) - indexes.EventEnc(serial).MarshalWrite(buf) + cmpBuf := new(bytes.Buffer) + indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf) - _, err := txn.Get(buf.Bytes()) + _, err := txn.Get(cmpBuf.Bytes()) if err == nil { - evtKeyExists = true + cmpKeyExists = true } return nil }) - if !evtKeyExists { - t.Errorf("Large event was not stored with evt prefix") + if !cmpKeyExists { + t.Errorf("Large event was not stored with cmp prefix (compact format)") } // Fetch and verify the event @@ -437,8 +416,8 @@ func TestInlineStorageMigration(t *testing.T) { } } -// BenchmarkInlineVsTraditionalStorage compares performance of inline vs traditional storage -func BenchmarkInlineVsTraditionalStorage(b *testing.B) { +// BenchmarkCompactStorage benchmarks the compact storage format performance +func BenchmarkCompactStorage(b *testing.B) { // Create a temporary directory for the database tempDir, err := os.MkdirTemp("", "bench-inline-db-*") if err != nil { @@ -501,7 +480,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) { } } - b.Run("FetchSmallEventsInline", func(b *testing.B) { + b.Run("FetchSmallEventsCompact", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { idx := i % len(smallSerials) @@ -509,7 +488,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) { } }) - b.Run("FetchLargeEventsTraditional", func(b *testing.B) { + b.Run("FetchLargeEventsCompact", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { idx := i % len(largeSerials) diff --git a/pkg/database/database.go b/pkg/database/database.go index 5bbc0e4..012a0e4 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -22,11 +22,10 @@ import ( // D implements the Database interface using Badger as the storage backend type D struct { - ctx context.Context - cancel context.CancelFunc - dataDir string - Logger *logger - inlineEventThreshold int // Configurable threshold for inline event storage + ctx context.Context + cancel context.CancelFunc + dataDir string + Logger *logger *badger.DB seq *badger.Sequence pubkeySeq *badger.Sequence // Sequence for pubkey serials @@ -51,13 +50,12 @@ func New( ) { // Create a default config for backward compatibility cfg := &DatabaseConfig{ - DataDir: dataDir, - LogLevel: logLevel, - BlockCacheMB: 1024, // Default 1024 MB - IndexCacheMB: 512, // Default 512 MB - QueryCacheSizeMB: 512, // Default 512 MB - QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes - InlineEventThreshold: 1024, // Default 1024 bytes + DataDir: dataDir, + LogLevel: logLevel, + BlockCacheMB: 1024, // Default 1024 MB + IndexCacheMB: 512, // Default 512 MB + QueryCacheSizeMB: 512, // Default 512 MB + QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes } return NewWithConfig(ctx, cancel, cfg) } @@ -86,10 +84,6 @@ func NewWithConfig( if queryCacheMaxAge == 0 { queryCacheMaxAge = 5 * time.Minute // Default 5 minutes } - inlineEventThreshold := cfg.InlineEventThreshold - if inlineEventThreshold == 0 { - inlineEventThreshold = 1024 // Default 1024 bytes - } // Serial cache configuration for compact event storage serialCachePubkeys := cfg.SerialCachePubkeys @@ -113,16 +107,15 @@ func NewWithConfig( queryCacheSize := int64(queryCacheSizeMB * 1024 * 1024) d = &D{ - ctx: ctx, - cancel: cancel, - dataDir: cfg.DataDir, - Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir), - inlineEventThreshold: inlineEventThreshold, - DB: nil, - seq: nil, - ready: make(chan struct{}), - queryCache: querycache.NewEventCache(queryCacheSize, queryCacheMaxAge), - serialCache: NewSerialCache(serialCachePubkeys, serialCacheEventIds), + ctx: ctx, + cancel: cancel, + dataDir: cfg.DataDir, + Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir), + DB: nil, + seq: nil, + ready: make(chan struct{}), + queryCache: querycache.NewEventCache(queryCacheSize, queryCacheMaxAge), + serialCache: NewSerialCache(serialCachePubkeys, serialCacheEventIds), } // Ensure the data directory exists diff --git a/pkg/database/factory.go b/pkg/database/factory.go index 33464cf..7276b75 100644 --- a/pkg/database/factory.go +++ b/pkg/database/factory.go @@ -18,11 +18,10 @@ type DatabaseConfig struct { LogLevel string // Badger-specific settings - BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB - IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB - QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB - QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE - InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD + BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB + IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB + QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB + QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE // Serial cache settings for compact event storage SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000) diff --git a/pkg/database/factory_wasm.go b/pkg/database/factory_wasm.go index 3560300..003bdec 100644 --- a/pkg/database/factory_wasm.go +++ b/pkg/database/factory_wasm.go @@ -18,11 +18,10 @@ type DatabaseConfig struct { LogLevel string // Badger-specific settings (not available in WASM) - BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB - IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB - QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB - QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE - InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD + BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB + IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB + QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB + QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE // Serial cache settings for compact event storage (Badger-specific) SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000) diff --git a/pkg/version/version b/pkg/version/version index 12b531b..a883ce3 100644 --- a/pkg/version/version +++ b/pkg/version/version @@ -1 +1 @@ -v0.33.0 \ No newline at end of file +v0.33.1 \ No newline at end of file