remove outdated configuration items for obsolete tail packing optimization
Some checks failed
Go / build-and-release (push) Has been cancelled
Some checks failed
Go / build-and-release (push) Has been cancelled
This commit is contained in:
@@ -179,7 +179,8 @@
|
||||
"Bash(CGO_ENABLED=0 LOG_LEVEL=trace go test:*)",
|
||||
"Bash(go vet:*)",
|
||||
"Bash(gofmt:*)",
|
||||
"Skill(cypher)"
|
||||
"Skill(cypher)",
|
||||
"Bash(git mv:*)"
|
||||
],
|
||||
"deny": [],
|
||||
"ask": []
|
||||
|
||||
@@ -156,7 +156,6 @@ export ORLY_QUERY_CACHE_MAX_AGE=5m # Cache expiry time
|
||||
export ORLY_DB_BLOCK_CACHE_MB=512 # Block cache size
|
||||
export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size
|
||||
export ORLY_DB_ZSTD_LEVEL=1 # ZSTD level: 0=off, 1=fast, 3=default, 9=best
|
||||
export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
|
||||
# Serial cache for compact event storage (Badger backend)
|
||||
export ORLY_SERIAL_CACHE_PUBKEYS=100000 # Max pubkeys to cache (~3.2MB memory)
|
||||
|
||||
@@ -101,7 +101,6 @@ type C struct {
|
||||
Neo4jPassword string `env:"ORLY_NEO4J_PASSWORD" default:"password" usage:"Neo4j authentication password (only used when ORLY_DB_TYPE=neo4j)"`
|
||||
|
||||
// Advanced database tuning
|
||||
InlineEventThreshold int `env:"ORLY_INLINE_EVENT_THRESHOLD" default:"1024" usage:"size threshold in bytes for inline event storage in Badger (0 to disable, typical values: 384-1024)"`
|
||||
SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"`
|
||||
SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
|
||||
|
||||
@@ -411,7 +410,6 @@ func (cfg *C) GetDatabaseConfigValues() (
|
||||
dataDir, logLevel string,
|
||||
blockCacheMB, indexCacheMB, queryCacheSizeMB int,
|
||||
queryCacheMaxAge time.Duration,
|
||||
inlineEventThreshold int,
|
||||
serialCachePubkeys, serialCacheEventIds int,
|
||||
zstdLevel int,
|
||||
neo4jURI, neo4jUser, neo4jPassword string,
|
||||
@@ -427,7 +425,6 @@ func (cfg *C) GetDatabaseConfigValues() (
|
||||
return cfg.DataDir, cfg.DBLogLevel,
|
||||
cfg.DBBlockCacheMB, cfg.DBIndexCacheMB, cfg.QueryCacheSizeMB,
|
||||
queryCacheMaxAge,
|
||||
cfg.InlineEventThreshold,
|
||||
cfg.SerialCachePubkeys, cfg.SerialCacheEventIds,
|
||||
cfg.DBZSTDLevel,
|
||||
cfg.Neo4jURI, cfg.Neo4jUser, cfg.Neo4jPassword
|
||||
|
||||
2
main.go
2
main.go
@@ -444,7 +444,6 @@ func makeDatabaseConfig(cfg *config.C) *database.DatabaseConfig {
|
||||
dataDir, logLevel,
|
||||
blockCacheMB, indexCacheMB, queryCacheSizeMB,
|
||||
queryCacheMaxAge,
|
||||
inlineEventThreshold,
|
||||
serialCachePubkeys, serialCacheEventIds,
|
||||
zstdLevel,
|
||||
neo4jURI, neo4jUser, neo4jPassword := cfg.GetDatabaseConfigValues()
|
||||
@@ -456,7 +455,6 @@ func makeDatabaseConfig(cfg *config.C) *database.DatabaseConfig {
|
||||
IndexCacheMB: indexCacheMB,
|
||||
QueryCacheSizeMB: queryCacheSizeMB,
|
||||
QueryCacheMaxAge: queryCacheMaxAge,
|
||||
InlineEventThreshold: inlineEventThreshold,
|
||||
SerialCachePubkeys: serialCachePubkeys,
|
||||
SerialCacheEventIds: serialCacheEventIds,
|
||||
ZSTDLevel: zstdLevel,
|
||||
|
||||
@@ -19,11 +19,12 @@ import (
|
||||
"git.mleku.dev/mleku/nostr/interfaces/signer/p8k"
|
||||
)
|
||||
|
||||
// TestInlineSmallEventStorage tests the Reiser4-inspired inline storage optimization
|
||||
// for small events (<=1024 bytes by default).
|
||||
func TestInlineSmallEventStorage(t *testing.T) {
|
||||
// TestCompactEventStorage tests the compact storage format (cmp prefix) which
|
||||
// replaced the old inline storage optimization (sev/evt prefixes).
|
||||
// All events are now stored in compact format regardless of size.
|
||||
func TestCompactEventStorage(t *testing.T) {
|
||||
// Create a temporary directory for the database
|
||||
tempDir, err := os.MkdirTemp("", "test-inline-db-*")
|
||||
tempDir, err := os.MkdirTemp("", "test-compact-db-*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temporary directory: %v", err)
|
||||
}
|
||||
@@ -46,8 +47,8 @@ func TestInlineSmallEventStorage(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Test Case 1: Small event (should use inline storage)
|
||||
t.Run("SmallEventInlineStorage", func(t *testing.T) {
|
||||
// Test Case 1: Small event (should use compact storage)
|
||||
t.Run("SmallEventCompactStorage", func(t *testing.T) {
|
||||
smallEvent := event.New()
|
||||
smallEvent.Kind = kind.TextNote.K
|
||||
smallEvent.CreatedAt = timestamp.Now().V
|
||||
@@ -65,49 +66,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
|
||||
t.Fatalf("Failed to save small event: %v", err)
|
||||
}
|
||||
|
||||
// Verify it was stored with sev prefix
|
||||
// Verify it was stored with cmp prefix
|
||||
serial, err := db.GetSerialById(smallEvent.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get serial for small event: %v", err)
|
||||
}
|
||||
|
||||
// Check that sev key exists
|
||||
sevKeyExists := false
|
||||
// Check that cmp key exists (compact format)
|
||||
cmpKeyExists := false
|
||||
db.View(func(txn *badger.Txn) error {
|
||||
smallBuf := new(bytes.Buffer)
|
||||
indexes.SmallEventEnc(serial).MarshalWrite(smallBuf)
|
||||
cmpBuf := new(bytes.Buffer)
|
||||
indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
|
||||
|
||||
opts := badger.DefaultIteratorOptions
|
||||
opts.Prefix = smallBuf.Bytes()
|
||||
it := txn.NewIterator(opts)
|
||||
defer it.Close()
|
||||
|
||||
it.Rewind()
|
||||
if it.Valid() {
|
||||
sevKeyExists = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if !sevKeyExists {
|
||||
t.Errorf("Small event was not stored with sev prefix")
|
||||
}
|
||||
|
||||
// Verify evt key does NOT exist for small event
|
||||
evtKeyExists := false
|
||||
db.View(func(txn *badger.Txn) error {
|
||||
buf := new(bytes.Buffer)
|
||||
indexes.EventEnc(serial).MarshalWrite(buf)
|
||||
|
||||
_, err := txn.Get(buf.Bytes())
|
||||
_, err := txn.Get(cmpBuf.Bytes())
|
||||
if err == nil {
|
||||
evtKeyExists = true
|
||||
cmpKeyExists = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if evtKeyExists {
|
||||
t.Errorf("Small event should not have evt key (should only use sev)")
|
||||
if !cmpKeyExists {
|
||||
t.Errorf("Small event was not stored with cmp prefix (compact format)")
|
||||
}
|
||||
|
||||
// Fetch and verify the event
|
||||
@@ -124,12 +103,12 @@ func TestInlineSmallEventStorage(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Test Case 2: Large event (should use traditional storage)
|
||||
t.Run("LargeEventTraditionalStorage", func(t *testing.T) {
|
||||
// Test Case 2: Large event (should also use compact storage)
|
||||
t.Run("LargeEventCompactStorage", func(t *testing.T) {
|
||||
largeEvent := event.New()
|
||||
largeEvent.Kind = kind.TextNote.K
|
||||
largeEvent.CreatedAt = timestamp.Now().V
|
||||
// Create content larger than 1024 bytes (the default inline storage threshold)
|
||||
// Create larger content
|
||||
largeContent := make([]byte, 1500)
|
||||
for i := range largeContent {
|
||||
largeContent[i] = 'x'
|
||||
@@ -148,27 +127,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
|
||||
t.Fatalf("Failed to save large event: %v", err)
|
||||
}
|
||||
|
||||
// Verify it was stored with evt prefix
|
||||
// Verify it was stored with cmp prefix (compact format)
|
||||
serial, err := db.GetSerialById(largeEvent.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get serial for large event: %v", err)
|
||||
}
|
||||
|
||||
// Check that evt key exists
|
||||
evtKeyExists := false
|
||||
// Check that cmp key exists
|
||||
cmpKeyExists := false
|
||||
db.View(func(txn *badger.Txn) error {
|
||||
buf := new(bytes.Buffer)
|
||||
indexes.EventEnc(serial).MarshalWrite(buf)
|
||||
cmpBuf := new(bytes.Buffer)
|
||||
indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
|
||||
|
||||
_, err := txn.Get(buf.Bytes())
|
||||
_, err := txn.Get(cmpBuf.Bytes())
|
||||
if err == nil {
|
||||
evtKeyExists = true
|
||||
cmpKeyExists = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if !evtKeyExists {
|
||||
t.Errorf("Large event was not stored with evt prefix")
|
||||
if !cmpKeyExists {
|
||||
t.Errorf("Large event was not stored with cmp prefix (compact format)")
|
||||
}
|
||||
|
||||
// Fetch and verify the event
|
||||
@@ -437,8 +416,8 @@ func TestInlineStorageMigration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkInlineVsTraditionalStorage compares performance of inline vs traditional storage
|
||||
func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
|
||||
// BenchmarkCompactStorage benchmarks the compact storage format performance
|
||||
func BenchmarkCompactStorage(b *testing.B) {
|
||||
// Create a temporary directory for the database
|
||||
tempDir, err := os.MkdirTemp("", "bench-inline-db-*")
|
||||
if err != nil {
|
||||
@@ -501,7 +480,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
b.Run("FetchSmallEventsInline", func(b *testing.B) {
|
||||
b.Run("FetchSmallEventsCompact", func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
idx := i % len(smallSerials)
|
||||
@@ -509,7 +488,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("FetchLargeEventsTraditional", func(b *testing.B) {
|
||||
b.Run("FetchLargeEventsCompact", func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
idx := i % len(largeSerials)
|
||||
@@ -26,7 +26,6 @@ type D struct {
|
||||
cancel context.CancelFunc
|
||||
dataDir string
|
||||
Logger *logger
|
||||
inlineEventThreshold int // Configurable threshold for inline event storage
|
||||
*badger.DB
|
||||
seq *badger.Sequence
|
||||
pubkeySeq *badger.Sequence // Sequence for pubkey serials
|
||||
@@ -57,7 +56,6 @@ func New(
|
||||
IndexCacheMB: 512, // Default 512 MB
|
||||
QueryCacheSizeMB: 512, // Default 512 MB
|
||||
QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes
|
||||
InlineEventThreshold: 1024, // Default 1024 bytes
|
||||
}
|
||||
return NewWithConfig(ctx, cancel, cfg)
|
||||
}
|
||||
@@ -86,10 +84,6 @@ func NewWithConfig(
|
||||
if queryCacheMaxAge == 0 {
|
||||
queryCacheMaxAge = 5 * time.Minute // Default 5 minutes
|
||||
}
|
||||
inlineEventThreshold := cfg.InlineEventThreshold
|
||||
if inlineEventThreshold == 0 {
|
||||
inlineEventThreshold = 1024 // Default 1024 bytes
|
||||
}
|
||||
|
||||
// Serial cache configuration for compact event storage
|
||||
serialCachePubkeys := cfg.SerialCachePubkeys
|
||||
@@ -117,7 +111,6 @@ func NewWithConfig(
|
||||
cancel: cancel,
|
||||
dataDir: cfg.DataDir,
|
||||
Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir),
|
||||
inlineEventThreshold: inlineEventThreshold,
|
||||
DB: nil,
|
||||
seq: nil,
|
||||
ready: make(chan struct{}),
|
||||
|
||||
@@ -22,7 +22,6 @@ type DatabaseConfig struct {
|
||||
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
|
||||
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
|
||||
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
|
||||
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
|
||||
|
||||
// Serial cache settings for compact event storage
|
||||
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)
|
||||
|
||||
@@ -22,7 +22,6 @@ type DatabaseConfig struct {
|
||||
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
|
||||
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
|
||||
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
|
||||
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
|
||||
|
||||
// Serial cache settings for compact event storage (Badger-specific)
|
||||
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)
|
||||
|
||||
@@ -1 +1 @@
|
||||
v0.33.0
|
||||
v0.33.1
|
||||
Reference in New Issue
Block a user