mleku/next.orly.dev
1
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

remove outdated configuration items for obsolete tail packing optimization
Some checks failed
Go / build-and-release (push) Has been cancelled
Details

Browse Source
This commit is contained in:
mleku
2025-12-03 21:24:43 +00:00
parent 54ead81791
commit 290fcbf8f0
9 changed files with 74 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
pkg/database/inline-storage_test.go → pkg/database/compact-storage_test.go
View File

@@ -19,11 +19,12 @@ import (
"git.mleku.dev/mleku/nostr/interfaces/signer/p8k"
)
// TestInlineSmallEventStorage tests the Reiser4-inspired inline storage optimization
// for small events (<=1024 bytes by default).
func TestInlineSmallEventStorage(t *testing.T) {
// TestCompactEventStorage tests the compact storage format (cmp prefix) which
// replaced the old inline storage optimization (sev/evt prefixes).
// All events are now stored in compact format regardless of size.
func TestCompactEventStorage(t *testing.T) {
// Create a temporary directory for the database
tempDir, err := os.MkdirTemp("", "test-inline-db-*")
tempDir, err := os.MkdirTemp("", "test-compact-db-*")
if err != nil {
t.Fatalf("Failed to create temporary directory: %v", err)
}
@@ -46,8 +47,8 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatal(err)
}
// Test Case 1: Small event (should use inline storage)
t.Run("SmallEventInlineStorage", func(t *testing.T) {
// Test Case 1: Small event (should use compact storage)
t.Run("SmallEventCompactStorage", func(t *testing.T) {
smallEvent := event.New()
smallEvent.Kind = kind.TextNote.K
smallEvent.CreatedAt = timestamp.Now().V
@@ -65,49 +66,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatalf("Failed to save small event: %v", err)
}
// Verify it was stored with sev prefix
// Verify it was stored with cmp prefix
serial, err := db.GetSerialById(smallEvent.ID)
if err != nil {
t.Fatalf("Failed to get serial for small event: %v", err)
}
// Check that sev key exists
sevKeyExists := false
// Check that cmp key exists (compact format)
cmpKeyExists := false
db.View(func(txn *badger.Txn) error {
smallBuf := new(bytes.Buffer)
indexes.SmallEventEnc(serial).MarshalWrite(smallBuf)
cmpBuf := new(bytes.Buffer)
indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
opts := badger.DefaultIteratorOptions
opts.Prefix = smallBuf.Bytes()
it := txn.NewIterator(opts)
defer it.Close()
it.Rewind()
if it.Valid() {
sevKeyExists = true
}
return nil
})
if !sevKeyExists {
t.Errorf("Small event was not stored with sev prefix")
}
// Verify evt key does NOT exist for small event
evtKeyExists := false
db.View(func(txn *badger.Txn) error {
buf := new(bytes.Buffer)
indexes.EventEnc(serial).MarshalWrite(buf)
_, err := txn.Get(buf.Bytes())
_, err := txn.Get(cmpBuf.Bytes())
if err == nil {
evtKeyExists = true
cmpKeyExists = true
}
return nil
})
if evtKeyExists {
t.Errorf("Small event should not have evt key (should only use sev)")
if !cmpKeyExists {
t.Errorf("Small event was not stored with cmp prefix (compact format)")
}
// Fetch and verify the event
@@ -124,12 +103,12 @@ func TestInlineSmallEventStorage(t *testing.T) {
}
})
// Test Case 2: Large event (should use traditional storage)
t.Run("LargeEventTraditionalStorage", func(t *testing.T) {
// Test Case 2: Large event (should also use compact storage)
t.Run("LargeEventCompactStorage", func(t *testing.T) {
largeEvent := event.New()
largeEvent.Kind = kind.TextNote.K
largeEvent.CreatedAt = timestamp.Now().V
// Create content larger than 1024 bytes (the default inline storage threshold)
// Create larger content
largeContent := make([]byte, 1500)
for i := range largeContent {
largeContent[i] = 'x'
@@ -148,27 +127,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatalf("Failed to save large event: %v", err)
}
// Verify it was stored with evt prefix
// Verify it was stored with cmp prefix (compact format)
serial, err := db.GetSerialById(largeEvent.ID)
if err != nil {
t.Fatalf("Failed to get serial for large event: %v", err)
}
// Check that evt key exists
evtKeyExists := false
// Check that cmp key exists
cmpKeyExists := false
db.View(func(txn *badger.Txn) error {
buf := new(bytes.Buffer)
indexes.EventEnc(serial).MarshalWrite(buf)
cmpBuf := new(bytes.Buffer)
indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
_, err := txn.Get(buf.Bytes())
_, err := txn.Get(cmpBuf.Bytes())
if err == nil {
evtKeyExists = true
cmpKeyExists = true
}
return nil
})
if !evtKeyExists {
t.Errorf("Large event was not stored with evt prefix")
if !cmpKeyExists {
t.Errorf("Large event was not stored with cmp prefix (compact format)")
}
// Fetch and verify the event
@@ -437,8 +416,8 @@ func TestInlineStorageMigration(t *testing.T) {
}
}
// BenchmarkInlineVsTraditionalStorage compares performance of inline vs traditional storage
func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
// BenchmarkCompactStorage benchmarks the compact storage format performance
func BenchmarkCompactStorage(b *testing.B) {
// Create a temporary directory for the database
tempDir, err := os.MkdirTemp("", "bench-inline-db-*")
if err != nil {
@@ -501,7 +480,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
}
}
b.Run("FetchSmallEventsInline", func(b *testing.B) {
b.Run("FetchSmallEventsCompact", func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
idx := i % len(smallSerials)
@@ -509,7 +488,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
}
})
b.Run("FetchLargeEventsTraditional", func(b *testing.B) {
b.Run("FetchLargeEventsCompact", func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
idx := i % len(largeSerials)

45
pkg/database/database.go
View File

@@ -22,11 +22,10 @@ import (
// D implements the Database interface using Badger as the storage backend
type D struct {
ctx context.Context
cancel context.CancelFunc
dataDir string
Logger *logger
inlineEventThreshold int // Configurable threshold for inline event storage
ctx context.Context
cancel context.CancelFunc
dataDir string
Logger *logger
*badger.DB
seq *badger.Sequence
pubkeySeq *badger.Sequence // Sequence for pubkey serials
@@ -51,13 +50,12 @@ func New(
) {
// Create a default config for backward compatibility
cfg := &DatabaseConfig{
DataDir: dataDir,
LogLevel: logLevel,
BlockCacheMB: 1024, // Default 1024 MB
IndexCacheMB: 512, // Default 512 MB
QueryCacheSizeMB: 512, // Default 512 MB
QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes
InlineEventThreshold: 1024, // Default 1024 bytes
DataDir: dataDir,
LogLevel: logLevel,
BlockCacheMB: 1024, // Default 1024 MB
IndexCacheMB: 512, // Default 512 MB
QueryCacheSizeMB: 512, // Default 512 MB
QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes
}
return NewWithConfig(ctx, cancel, cfg)
}
@@ -86,10 +84,6 @@ func NewWithConfig(
if queryCacheMaxAge == 0 {
queryCacheMaxAge = 5 * time.Minute // Default 5 minutes
}
inlineEventThreshold := cfg.InlineEventThreshold
if inlineEventThreshold == 0 {
inlineEventThreshold = 1024 // Default 1024 bytes
}
// Serial cache configuration for compact event storage
serialCachePubkeys := cfg.SerialCachePubkeys
@@ -113,16 +107,15 @@ func NewWithConfig(
queryCacheSize := int64(queryCacheSizeMB * 1024 * 1024)
d = &D{
ctx: ctx,
cancel: cancel,
dataDir: cfg.DataDir,
Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir),
inlineEventThreshold: inlineEventThreshold,
DB: nil,
seq: nil,
ready: make(chan struct{}),
queryCache: querycache.NewEventCache(queryCacheSize, queryCacheMaxAge),
serialCache: NewSerialCache(serialCachePubkeys, serialCacheEventIds),
ctx: ctx,
cancel: cancel,
dataDir: cfg.DataDir,
Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir),
DB: nil,
seq: nil,
ready: make(chan struct{}),
queryCache: querycache.NewEventCache(queryCacheSize, queryCacheMaxAge),
serialCache: NewSerialCache(serialCachePubkeys, serialCacheEventIds),
}
// Ensure the data directory exists

9
pkg/database/factory.go
View File

@@ -18,11 +18,10 @@ type DatabaseConfig struct {
LogLevel string
// Badger-specific settings
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
// Serial cache settings for compact event storage
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)

9
pkg/database/factory_wasm.go
View File

@@ -18,11 +18,10 @@ type DatabaseConfig struct {
LogLevel string
// Badger-specific settings (not available in WASM)
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
// Serial cache settings for compact event storage (Badger-specific)
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)