mleku/next.orly.dev
1
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

remove outdated configuration items for obsolete tail packing optimization
Some checks failed
Go / build-and-release (push) Has been cancelled
Details

Browse Source
This commit is contained in:
mleku
2025-12-03 21:24:43 +00:00
parent 54ead81791
commit 290fcbf8f0
9 changed files with 74 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.claude/settings.local.json
View File

@@ -179,7 +179,8 @@
"Bash(CGO_ENABLED=0 LOG_LEVEL=trace go test:*)", "Bash(CGO_ENABLED=0 LOG_LEVEL=trace go test:*)",
"Bash(go vet:*)", "Bash(go vet:*)",
"Bash(gofmt:*)", "Bash(gofmt:*)",
"Skill(cypher)" "Skill(cypher)",
"Bash(git mv:*)"
], ],
"deny": [], "deny": [],
"ask": [] "ask": []

1
CLAUDE.md
View File

@@ -156,7 +156,6 @@ export ORLY_QUERY_CACHE_MAX_AGE=5m # Cache expiry time
export ORLY_DB_BLOCK_CACHE_MB=512 # Block cache size export ORLY_DB_BLOCK_CACHE_MB=512 # Block cache size
export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size
export ORLY_DB_ZSTD_LEVEL=1 # ZSTD level: 0=off, 1=fast, 3=default, 9=best export ORLY_DB_ZSTD_LEVEL=1 # ZSTD level: 0=off, 1=fast, 3=default, 9=best
export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
# Serial cache for compact event storage (Badger backend) # Serial cache for compact event storage (Badger backend)
export ORLY_SERIAL_CACHE_PUBKEYS=100000 # Max pubkeys to cache (~3.2MB memory) export ORLY_SERIAL_CACHE_PUBKEYS=100000 # Max pubkeys to cache (~3.2MB memory)

3
app/config/config.go
View File

@@ -101,7 +101,6 @@ type C struct {
Neo4jPassword string `env:"ORLY_NEO4J_PASSWORD" default:"password" usage:"Neo4j authentication password (only used when ORLY_DB_TYPE=neo4j)"` Neo4jPassword string `env:"ORLY_NEO4J_PASSWORD" default:"password" usage:"Neo4j authentication password (only used when ORLY_DB_TYPE=neo4j)"`
// Advanced database tuning // Advanced database tuning
InlineEventThreshold int `env:"ORLY_INLINE_EVENT_THRESHOLD" default:"1024" usage:"size threshold in bytes for inline event storage in Badger (0 to disable, typical values: 384-1024)"`
SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"` SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"`
SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"` SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
@@ -411,7 +410,6 @@ func (cfg *C) GetDatabaseConfigValues() (
dataDir, logLevel string, dataDir, logLevel string,
blockCacheMB, indexCacheMB, queryCacheSizeMB int, blockCacheMB, indexCacheMB, queryCacheSizeMB int,
queryCacheMaxAge time.Duration, queryCacheMaxAge time.Duration,
inlineEventThreshold int,
serialCachePubkeys, serialCacheEventIds int, serialCachePubkeys, serialCacheEventIds int,
zstdLevel int, zstdLevel int,
neo4jURI, neo4jUser, neo4jPassword string, neo4jURI, neo4jUser, neo4jPassword string,
@@ -427,7 +425,6 @@ func (cfg *C) GetDatabaseConfigValues() (
return cfg.DataDir, cfg.DBLogLevel, return cfg.DataDir, cfg.DBLogLevel,
cfg.DBBlockCacheMB, cfg.DBIndexCacheMB, cfg.QueryCacheSizeMB, cfg.DBBlockCacheMB, cfg.DBIndexCacheMB, cfg.QueryCacheSizeMB,
queryCacheMaxAge, queryCacheMaxAge,
cfg.InlineEventThreshold,
cfg.SerialCachePubkeys, cfg.SerialCacheEventIds, cfg.SerialCachePubkeys, cfg.SerialCacheEventIds,
cfg.DBZSTDLevel, cfg.DBZSTDLevel,
cfg.Neo4jURI, cfg.Neo4jUser, cfg.Neo4jPassword cfg.Neo4jURI, cfg.Neo4jUser, cfg.Neo4jPassword

2
main.go
View File

@@ -444,7 +444,6 @@ func makeDatabaseConfig(cfg *config.C) *database.DatabaseConfig {
dataDir, logLevel, dataDir, logLevel,
blockCacheMB, indexCacheMB, queryCacheSizeMB, blockCacheMB, indexCacheMB, queryCacheSizeMB,
queryCacheMaxAge, queryCacheMaxAge,
inlineEventThreshold,
serialCachePubkeys, serialCacheEventIds, serialCachePubkeys, serialCacheEventIds,
zstdLevel, zstdLevel,
neo4jURI, neo4jUser, neo4jPassword := cfg.GetDatabaseConfigValues() neo4jURI, neo4jUser, neo4jPassword := cfg.GetDatabaseConfigValues()
@@ -456,7 +455,6 @@ func makeDatabaseConfig(cfg *config.C) *database.DatabaseConfig {
IndexCacheMB: indexCacheMB, IndexCacheMB: indexCacheMB,
QueryCacheSizeMB: queryCacheSizeMB, QueryCacheSizeMB: queryCacheSizeMB,
QueryCacheMaxAge: queryCacheMaxAge, QueryCacheMaxAge: queryCacheMaxAge,
InlineEventThreshold: inlineEventThreshold,
SerialCachePubkeys: serialCachePubkeys, SerialCachePubkeys: serialCachePubkeys,
SerialCacheEventIds: serialCacheEventIds, SerialCacheEventIds: serialCacheEventIds,
ZSTDLevel: zstdLevel, ZSTDLevel: zstdLevel,

85
pkg/database/inline-storage_test.go → pkg/database/compact-storage_test.go
View File

@@ -19,11 +19,12 @@ import (
"git.mleku.dev/mleku/nostr/interfaces/signer/p8k" "git.mleku.dev/mleku/nostr/interfaces/signer/p8k"
) )
// TestInlineSmallEventStorage tests the Reiser4-inspired inline storage optimization // TestCompactEventStorage tests the compact storage format (cmp prefix) which
// for small events (<=1024 bytes by default). // replaced the old inline storage optimization (sev/evt prefixes).
func TestInlineSmallEventStorage(t *testing.T) { // All events are now stored in compact format regardless of size.
func TestCompactEventStorage(t *testing.T) {
// Create a temporary directory for the database // Create a temporary directory for the database
tempDir, err := os.MkdirTemp("", "test-inline-db-*") tempDir, err := os.MkdirTemp("", "test-compact-db-*")
if err != nil { if err != nil {
t.Fatalf("Failed to create temporary directory: %v", err) t.Fatalf("Failed to create temporary directory: %v", err)
} }
@@ -46,8 +47,8 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Test Case 1: Small event (should use inline storage) // Test Case 1: Small event (should use compact storage)
t.Run("SmallEventInlineStorage", func(t *testing.T) { t.Run("SmallEventCompactStorage", func(t *testing.T) {
smallEvent := event.New() smallEvent := event.New()
smallEvent.Kind = kind.TextNote.K smallEvent.Kind = kind.TextNote.K
smallEvent.CreatedAt = timestamp.Now().V smallEvent.CreatedAt = timestamp.Now().V
@@ -65,49 +66,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatalf("Failed to save small event: %v", err) t.Fatalf("Failed to save small event: %v", err)
} }
// Verify it was stored with sev prefix // Verify it was stored with cmp prefix
serial, err := db.GetSerialById(smallEvent.ID) serial, err := db.GetSerialById(smallEvent.ID)
if err != nil { if err != nil {
t.Fatalf("Failed to get serial for small event: %v", err) t.Fatalf("Failed to get serial for small event: %v", err)
} }
// Check that sev key exists // Check that cmp key exists (compact format)
sevKeyExists := false cmpKeyExists := false
db.View(func(txn *badger.Txn) error { db.View(func(txn *badger.Txn) error {
smallBuf := new(bytes.Buffer) cmpBuf := new(bytes.Buffer)
indexes.SmallEventEnc(serial).MarshalWrite(smallBuf) indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
opts := badger.DefaultIteratorOptions _, err := txn.Get(cmpBuf.Bytes())
opts.Prefix = smallBuf.Bytes()
it := txn.NewIterator(opts)
defer it.Close()
it.Rewind()
if it.Valid() {
sevKeyExists = true
}
return nil
})
if !sevKeyExists {
t.Errorf("Small event was not stored with sev prefix")
}
// Verify evt key does NOT exist for small event
evtKeyExists := false
db.View(func(txn *badger.Txn) error {
buf := new(bytes.Buffer)
indexes.EventEnc(serial).MarshalWrite(buf)
_, err := txn.Get(buf.Bytes())
if err == nil { if err == nil {
evtKeyExists = true cmpKeyExists = true
} }
return nil return nil
}) })
if evtKeyExists { if !cmpKeyExists {
t.Errorf("Small event should not have evt key (should only use sev)") t.Errorf("Small event was not stored with cmp prefix (compact format)")
} }
// Fetch and verify the event // Fetch and verify the event
@@ -124,12 +103,12 @@ func TestInlineSmallEventStorage(t *testing.T) {
} }
}) })
// Test Case 2: Large event (should use traditional storage) // Test Case 2: Large event (should also use compact storage)
t.Run("LargeEventTraditionalStorage", func(t *testing.T) { t.Run("LargeEventCompactStorage", func(t *testing.T) {
largeEvent := event.New() largeEvent := event.New()
largeEvent.Kind = kind.TextNote.K largeEvent.Kind = kind.TextNote.K
largeEvent.CreatedAt = timestamp.Now().V largeEvent.CreatedAt = timestamp.Now().V
// Create content larger than 1024 bytes (the default inline storage threshold) // Create larger content
largeContent := make([]byte, 1500) largeContent := make([]byte, 1500)
for i := range largeContent { for i := range largeContent {
largeContent[i] = 'x' largeContent[i] = 'x'
@@ -148,27 +127,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatalf("Failed to save large event: %v", err) t.Fatalf("Failed to save large event: %v", err)
} }
// Verify it was stored with evt prefix // Verify it was stored with cmp prefix (compact format)
serial, err := db.GetSerialById(largeEvent.ID) serial, err := db.GetSerialById(largeEvent.ID)
if err != nil { if err != nil {
t.Fatalf("Failed to get serial for large event: %v", err) t.Fatalf("Failed to get serial for large event: %v", err)
} }
// Check that evt key exists // Check that cmp key exists
evtKeyExists := false cmpKeyExists := false
db.View(func(txn *badger.Txn) error { db.View(func(txn *badger.Txn) error {
buf := new(bytes.Buffer) cmpBuf := new(bytes.Buffer)
indexes.EventEnc(serial).MarshalWrite(buf) indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
_, err := txn.Get(buf.Bytes()) _, err := txn.Get(cmpBuf.Bytes())
if err == nil { if err == nil {
evtKeyExists = true cmpKeyExists = true
} }
return nil return nil
}) })
if !evtKeyExists { if !cmpKeyExists {
t.Errorf("Large event was not stored with evt prefix") t.Errorf("Large event was not stored with cmp prefix (compact format)")
} }
// Fetch and verify the event // Fetch and verify the event
@@ -437,8 +416,8 @@ func TestInlineStorageMigration(t *testing.T) {
} }
} }
// BenchmarkInlineVsTraditionalStorage compares performance of inline vs traditional storage // BenchmarkCompactStorage benchmarks the compact storage format performance
func BenchmarkInlineVsTraditionalStorage(b *testing.B) { func BenchmarkCompactStorage(b *testing.B) {
// Create a temporary directory for the database // Create a temporary directory for the database
tempDir, err := os.MkdirTemp("", "bench-inline-db-*") tempDir, err := os.MkdirTemp("", "bench-inline-db-*")
if err != nil { if err != nil {
@@ -501,7 +480,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
} }
} }
b.Run("FetchSmallEventsInline", func(b *testing.B) { b.Run("FetchSmallEventsCompact", func(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
idx := i % len(smallSerials) idx := i % len(smallSerials)
@@ -509,7 +488,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
} }
}) })
b.Run("FetchLargeEventsTraditional", func(b *testing.B) { b.Run("FetchLargeEventsCompact", func(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
idx := i % len(largeSerials) idx := i % len(largeSerials)

7
pkg/database/database.go
View File

@@ -26,7 +26,6 @@ type D struct {
cancel context.CancelFunc cancel context.CancelFunc
dataDir string dataDir string
Logger *logger Logger *logger
inlineEventThreshold int // Configurable threshold for inline event storage
*badger.DB *badger.DB
seq *badger.Sequence seq *badger.Sequence
pubkeySeq *badger.Sequence // Sequence for pubkey serials pubkeySeq *badger.Sequence // Sequence for pubkey serials
@@ -57,7 +56,6 @@ func New(
IndexCacheMB: 512, // Default 512 MB IndexCacheMB: 512, // Default 512 MB
QueryCacheSizeMB: 512, // Default 512 MB QueryCacheSizeMB: 512, // Default 512 MB
QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes
InlineEventThreshold: 1024, // Default 1024 bytes
} }
return NewWithConfig(ctx, cancel, cfg) return NewWithConfig(ctx, cancel, cfg)
} }
@@ -86,10 +84,6 @@ func NewWithConfig(
if queryCacheMaxAge == 0 { if queryCacheMaxAge == 0 {
queryCacheMaxAge = 5 * time.Minute // Default 5 minutes queryCacheMaxAge = 5 * time.Minute // Default 5 minutes
} }
inlineEventThreshold := cfg.InlineEventThreshold
if inlineEventThreshold == 0 {
inlineEventThreshold = 1024 // Default 1024 bytes
}
// Serial cache configuration for compact event storage // Serial cache configuration for compact event storage
serialCachePubkeys := cfg.SerialCachePubkeys serialCachePubkeys := cfg.SerialCachePubkeys
@@ -117,7 +111,6 @@ func NewWithConfig(
cancel: cancel, cancel: cancel,
dataDir: cfg.DataDir, dataDir: cfg.DataDir,
Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir), Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir),
inlineEventThreshold: inlineEventThreshold,
DB: nil, DB: nil,
seq: nil, seq: nil,
ready: make(chan struct{}), ready: make(chan struct{}),

1
pkg/database/factory.go
View File

@@ -22,7 +22,6 @@ type DatabaseConfig struct {
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
// Serial cache settings for compact event storage // Serial cache settings for compact event storage
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000) SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)

1
pkg/database/factory_wasm.go
View File

@@ -22,7 +22,6 @@ type DatabaseConfig struct {
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
// Serial cache settings for compact event storage (Badger-specific) // Serial cache settings for compact event storage (Badger-specific)
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000) SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)

2
pkg/version/version
View File

@@ -1 +1 @@
v0.33.0 v0.33.1