mleku/next.orly.dev
1
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

remove outdated configuration items for obsolete tail packing optimization
Some checks failed
Go / build-and-release (push) Has been cancelled
Details

Browse Source
This commit is contained in:
mleku
2025-12-03 21:24:43 +00:00
parent 54ead81791
commit 290fcbf8f0
9 changed files with 74 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.claude/settings.local.json
View File

@@ -179,7 +179,8 @@
"Bash(CGO_ENABLED=0 LOG_LEVEL=trace go test:*)",
"Bash(go vet:*)",
"Bash(gofmt:*)",
"Skill(cypher)"
"Skill(cypher)",
"Bash(git mv:*)"
],
"deny": [],
"ask": []

1
CLAUDE.md
View File

@@ -156,7 +156,6 @@ export ORLY_QUERY_CACHE_MAX_AGE=5m # Cache expiry time
export ORLY_DB_BLOCK_CACHE_MB=512 # Block cache size
export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size
export ORLY_DB_ZSTD_LEVEL=1 # ZSTD level: 0=off, 1=fast, 3=default, 9=best
export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
# Serial cache for compact event storage (Badger backend)
export ORLY_SERIAL_CACHE_PUBKEYS=100000 # Max pubkeys to cache (~3.2MB memory)

3
app/config/config.go
View File

@@ -101,7 +101,6 @@ type C struct {
Neo4jPassword string `env:"ORLY_NEO4J_PASSWORD" default:"password" usage:"Neo4j authentication password (only used when ORLY_DB_TYPE=neo4j)"`
// Advanced database tuning
InlineEventThreshold int `env:"ORLY_INLINE_EVENT_THRESHOLD" default:"1024" usage:"size threshold in bytes for inline event storage in Badger (0 to disable, typical values: 384-1024)"`
SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"`
SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
@@ -411,7 +410,6 @@ func (cfg *C) GetDatabaseConfigValues() (
dataDir, logLevel string,
blockCacheMB, indexCacheMB, queryCacheSizeMB int,
queryCacheMaxAge time.Duration,
inlineEventThreshold int,
serialCachePubkeys, serialCacheEventIds int,
zstdLevel int,
neo4jURI, neo4jUser, neo4jPassword string,
@@ -427,7 +425,6 @@ func (cfg *C) GetDatabaseConfigValues() (
return cfg.DataDir, cfg.DBLogLevel,
cfg.DBBlockCacheMB, cfg.DBIndexCacheMB, cfg.QueryCacheSizeMB,
queryCacheMaxAge,
cfg.InlineEventThreshold,
cfg.SerialCachePubkeys, cfg.SerialCacheEventIds,
cfg.DBZSTDLevel,
cfg.Neo4jURI, cfg.Neo4jUser, cfg.Neo4jPassword

26
main.go
View File

@@ -444,24 +444,22 @@ func makeDatabaseConfig(cfg *config.C) *database.DatabaseConfig {
dataDir, logLevel,
blockCacheMB, indexCacheMB, queryCacheSizeMB,
queryCacheMaxAge,
inlineEventThreshold,
serialCachePubkeys, serialCacheEventIds,
zstdLevel,
neo4jURI, neo4jUser, neo4jPassword := cfg.GetDatabaseConfigValues()
return &database.DatabaseConfig{
DataDir: dataDir,
LogLevel: logLevel,
BlockCacheMB: blockCacheMB,
IndexCacheMB: indexCacheMB,
QueryCacheSizeMB: queryCacheSizeMB,
QueryCacheMaxAge: queryCacheMaxAge,
InlineEventThreshold: inlineEventThreshold,
SerialCachePubkeys: serialCachePubkeys,
SerialCacheEventIds: serialCacheEventIds,
ZSTDLevel: zstdLevel,
Neo4jURI: neo4jURI,
Neo4jUser: neo4jUser,
Neo4jPassword: neo4jPassword,
DataDir: dataDir,
LogLevel: logLevel,
BlockCacheMB: blockCacheMB,
IndexCacheMB: indexCacheMB,
QueryCacheSizeMB: queryCacheSizeMB,
QueryCacheMaxAge: queryCacheMaxAge,
SerialCachePubkeys: serialCachePubkeys,
SerialCacheEventIds: serialCacheEventIds,
ZSTDLevel: zstdLevel,
Neo4jURI: neo4jURI,
Neo4jUser: neo4jUser,
Neo4jPassword: neo4jPassword,
}
}

85
pkg/database/inline-storage_test.go → pkg/database/compact-storage_test.go
View File

@@ -19,11 +19,12 @@ import (
"git.mleku.dev/mleku/nostr/interfaces/signer/p8k"
)
// TestInlineSmallEventStorage tests the Reiser4-inspired inline storage optimization
// for small events (<=1024 bytes by default).
func TestInlineSmallEventStorage(t *testing.T) {
// TestCompactEventStorage tests the compact storage format (cmp prefix) which
// replaced the old inline storage optimization (sev/evt prefixes).
// All events are now stored in compact format regardless of size.
func TestCompactEventStorage(t *testing.T) {
// Create a temporary directory for the database
tempDir, err := os.MkdirTemp("", "test-inline-db-*")
tempDir, err := os.MkdirTemp("", "test-compact-db-*")
if err != nil {
t.Fatalf("Failed to create temporary directory: %v", err)
}
@@ -46,8 +47,8 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatal(err)
}
// Test Case 1: Small event (should use inline storage)
t.Run("SmallEventInlineStorage", func(t *testing.T) {
// Test Case 1: Small event (should use compact storage)
t.Run("SmallEventCompactStorage", func(t *testing.T) {
smallEvent := event.New()
smallEvent.Kind = kind.TextNote.K
smallEvent.CreatedAt = timestamp.Now().V
@@ -65,49 +66,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatalf("Failed to save small event: %v", err)
}
// Verify it was stored with sev prefix
// Verify it was stored with cmp prefix
serial, err := db.GetSerialById(smallEvent.ID)
if err != nil {
t.Fatalf("Failed to get serial for small event: %v", err)
}
// Check that sev key exists
sevKeyExists := false
// Check that cmp key exists (compact format)
cmpKeyExists := false
db.View(func(txn *badger.Txn) error {
smallBuf := new(bytes.Buffer)
indexes.SmallEventEnc(serial).MarshalWrite(smallBuf)
cmpBuf := new(bytes.Buffer)
indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
opts := badger.DefaultIteratorOptions
opts.Prefix = smallBuf.Bytes()
it := txn.NewIterator(opts)
defer it.Close()
it.Rewind()
if it.Valid() {
sevKeyExists = true
}
return nil
})
if !sevKeyExists {
t.Errorf("Small event was not stored with sev prefix")
}
// Verify evt key does NOT exist for small event
evtKeyExists := false
db.View(func(txn *badger.Txn) error {
buf := new(bytes.Buffer)
indexes.EventEnc(serial).MarshalWrite(buf)
_, err := txn.Get(buf.Bytes())
_, err := txn.Get(cmpBuf.Bytes())
if err == nil {
evtKeyExists = true
cmpKeyExists = true
}
return nil
})
if evtKeyExists {
t.Errorf("Small event should not have evt key (should only use sev)")
if !cmpKeyExists {
t.Errorf("Small event was not stored with cmp prefix (compact format)")
}
// Fetch and verify the event
@@ -124,12 +103,12 @@ func TestInlineSmallEventStorage(t *testing.T) {
}
})
// Test Case 2: Large event (should use traditional storage)
t.Run("LargeEventTraditionalStorage", func(t *testing.T) {
// Test Case 2: Large event (should also use compact storage)
t.Run("LargeEventCompactStorage", func(t *testing.T) {
largeEvent := event.New()
largeEvent.Kind = kind.TextNote.K
largeEvent.CreatedAt = timestamp.Now().V
// Create content larger than 1024 bytes (the default inline storage threshold)
// Create larger content
largeContent := make([]byte, 1500)
for i := range largeContent {
largeContent[i] = 'x'
@@ -148,27 +127,27 @@ func TestInlineSmallEventStorage(t *testing.T) {
t.Fatalf("Failed to save large event: %v", err)
}
// Verify it was stored with evt prefix
// Verify it was stored with cmp prefix (compact format)
serial, err := db.GetSerialById(largeEvent.ID)
if err != nil {
t.Fatalf("Failed to get serial for large event: %v", err)
}
// Check that evt key exists
evtKeyExists := false
// Check that cmp key exists
cmpKeyExists := false
db.View(func(txn *badger.Txn) error {
buf := new(bytes.Buffer)
indexes.EventEnc(serial).MarshalWrite(buf)
cmpBuf := new(bytes.Buffer)
indexes.CompactEventEnc(serial).MarshalWrite(cmpBuf)
_, err := txn.Get(buf.Bytes())
_, err := txn.Get(cmpBuf.Bytes())
if err == nil {
evtKeyExists = true
cmpKeyExists = true
}
return nil
})
if !evtKeyExists {
t.Errorf("Large event was not stored with evt prefix")
if !cmpKeyExists {
t.Errorf("Large event was not stored with cmp prefix (compact format)")
}
// Fetch and verify the event
@@ -437,8 +416,8 @@ func TestInlineStorageMigration(t *testing.T) {
}
}
// BenchmarkInlineVsTraditionalStorage compares performance of inline vs traditional storage
func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
// BenchmarkCompactStorage benchmarks the compact storage format performance
func BenchmarkCompactStorage(b *testing.B) {
// Create a temporary directory for the database
tempDir, err := os.MkdirTemp("", "bench-inline-db-*")
if err != nil {
@@ -501,7 +480,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
}
}
b.Run("FetchSmallEventsInline", func(b *testing.B) {
b.Run("FetchSmallEventsCompact", func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
idx := i % len(smallSerials)
@@ -509,7 +488,7 @@ func BenchmarkInlineVsTraditionalStorage(b *testing.B) {
}
})
b.Run("FetchLargeEventsTraditional", func(b *testing.B) {
b.Run("FetchLargeEventsCompact", func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
idx := i % len(largeSerials)

45
pkg/database/database.go
View File

@@ -22,11 +22,10 @@ import (
// D implements the Database interface using Badger as the storage backend
type D struct {
ctx context.Context
cancel context.CancelFunc
dataDir string
Logger *logger
inlineEventThreshold int // Configurable threshold for inline event storage
ctx context.Context
cancel context.CancelFunc
dataDir string
Logger *logger
*badger.DB
seq *badger.Sequence
pubkeySeq *badger.Sequence // Sequence for pubkey serials
@@ -51,13 +50,12 @@ func New(
) {
// Create a default config for backward compatibility
cfg := &DatabaseConfig{
DataDir: dataDir,
LogLevel: logLevel,
BlockCacheMB: 1024, // Default 1024 MB
IndexCacheMB: 512, // Default 512 MB
QueryCacheSizeMB: 512, // Default 512 MB
QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes
InlineEventThreshold: 1024, // Default 1024 bytes
DataDir: dataDir,
LogLevel: logLevel,
BlockCacheMB: 1024, // Default 1024 MB
IndexCacheMB: 512, // Default 512 MB
QueryCacheSizeMB: 512, // Default 512 MB
QueryCacheMaxAge: 5 * time.Minute, // Default 5 minutes
}
return NewWithConfig(ctx, cancel, cfg)
}
@@ -86,10 +84,6 @@ func NewWithConfig(
if queryCacheMaxAge == 0 {
queryCacheMaxAge = 5 * time.Minute // Default 5 minutes
}
inlineEventThreshold := cfg.InlineEventThreshold
if inlineEventThreshold == 0 {
inlineEventThreshold = 1024 // Default 1024 bytes
}
// Serial cache configuration for compact event storage
serialCachePubkeys := cfg.SerialCachePubkeys
@@ -113,16 +107,15 @@ func NewWithConfig(
queryCacheSize := int64(queryCacheSizeMB * 1024 * 1024)
d = &D{
ctx: ctx,
cancel: cancel,
dataDir: cfg.DataDir,
Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir),
inlineEventThreshold: inlineEventThreshold,
DB: nil,
seq: nil,
ready: make(chan struct{}),
queryCache: querycache.NewEventCache(queryCacheSize, queryCacheMaxAge),
serialCache: NewSerialCache(serialCachePubkeys, serialCacheEventIds),
ctx: ctx,
cancel: cancel,
dataDir: cfg.DataDir,
Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir),
DB: nil,
seq: nil,
ready: make(chan struct{}),
queryCache: querycache.NewEventCache(queryCacheSize, queryCacheMaxAge),
serialCache: NewSerialCache(serialCachePubkeys, serialCacheEventIds),
}
// Ensure the data directory exists

9
pkg/database/factory.go
View File

@@ -18,11 +18,10 @@ type DatabaseConfig struct {
LogLevel string
// Badger-specific settings
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
// Serial cache settings for compact event storage
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)

9
pkg/database/factory_wasm.go
View File

@@ -18,11 +18,10 @@ type DatabaseConfig struct {
LogLevel string
// Badger-specific settings (not available in WASM)
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
InlineEventThreshold int // ORLY_INLINE_EVENT_THRESHOLD
BlockCacheMB int // ORLY_DB_BLOCK_CACHE_MB
IndexCacheMB int // ORLY_DB_INDEX_CACHE_MB
QueryCacheSizeMB int // ORLY_QUERY_CACHE_SIZE_MB
QueryCacheMaxAge time.Duration // ORLY_QUERY_CACHE_MAX_AGE
// Serial cache settings for compact event storage (Badger-specific)
SerialCachePubkeys int // ORLY_SERIAL_CACHE_PUBKEYS - max pubkeys to cache (default: 100000)

2
pkg/version/version
View File

@@ -1 +1 @@
v0.33.0
v0.33.1