//go:build !(js && wasm) package database import ( "bytes" "sync/atomic" "github.com/dgraph-io/badger/v4" "lol.mleku.dev/chk" "lol.mleku.dev/log" "next.orly.dev/pkg/database/indexes" ) // CompactStorageStats holds statistics about compact vs legacy storage. type CompactStorageStats struct { // Event counts CompactEvents int64 // Number of events in compact format (cmp prefix) LegacyEvents int64 // Number of events in legacy format (evt/sev prefixes) TotalEvents int64 // Total events // Storage sizes CompactBytes int64 // Total bytes used by compact format LegacyBytes int64 // Total bytes used by legacy format (would be used without compact) // Savings BytesSaved int64 // Bytes saved by using compact format PercentSaved float64 // Percentage of space saved AverageCompact float64 // Average compact event size AverageLegacy float64 // Average legacy event size (estimated) // Serial mappings SerialEventIdEntries int64 // Number of sei (serial -> event ID) mappings SerialEventIdBytes int64 // Bytes used by sei mappings } // CompactStorageStats calculates storage statistics for compact event storage. // This scans the database to provide accurate metrics on space savings. func (d *D) CompactStorageStats() (stats CompactStorageStats, err error) { if err = d.View(func(txn *badger.Txn) error { // Count compact events (cmp prefix) cmpPrf := new(bytes.Buffer) if err = indexes.CompactEventEnc(nil).MarshalWrite(cmpPrf); chk.E(err) { return err } it := txn.NewIterator(badger.IteratorOptions{Prefix: cmpPrf.Bytes()}) for it.Rewind(); it.Valid(); it.Next() { item := it.Item() stats.CompactEvents++ stats.CompactBytes += int64(len(item.Key())) + int64(item.ValueSize()) } it.Close() // Count legacy evt entries evtPrf := new(bytes.Buffer) if err = indexes.EventEnc(nil).MarshalWrite(evtPrf); chk.E(err) { return err } it = txn.NewIterator(badger.IteratorOptions{Prefix: evtPrf.Bytes()}) for it.Rewind(); it.Valid(); it.Next() { item := it.Item() stats.LegacyEvents++ stats.LegacyBytes += int64(len(item.Key())) + int64(item.ValueSize()) } it.Close() // Count legacy sev entries sevPrf := new(bytes.Buffer) if err = indexes.SmallEventEnc(nil).MarshalWrite(sevPrf); chk.E(err) { return err } it = txn.NewIterator(badger.IteratorOptions{Prefix: sevPrf.Bytes()}) for it.Rewind(); it.Valid(); it.Next() { item := it.Item() stats.LegacyEvents++ stats.LegacyBytes += int64(len(item.Key())) // sev stores data in key } it.Close() // Count SerialEventId mappings (sei prefix) seiPrf := new(bytes.Buffer) if err = indexes.SerialEventIdEnc(nil).MarshalWrite(seiPrf); chk.E(err) { return err } it = txn.NewIterator(badger.IteratorOptions{Prefix: seiPrf.Bytes()}) for it.Rewind(); it.Valid(); it.Next() { item := it.Item() stats.SerialEventIdEntries++ stats.SerialEventIdBytes += int64(len(item.Key())) + int64(item.ValueSize()) } it.Close() return nil }); chk.E(err) { return } stats.TotalEvents = stats.CompactEvents + stats.LegacyEvents // Calculate averages if stats.CompactEvents > 0 { stats.AverageCompact = float64(stats.CompactBytes) / float64(stats.CompactEvents) } if stats.LegacyEvents > 0 { stats.AverageLegacy = float64(stats.LegacyBytes) / float64(stats.LegacyEvents) } // Estimate savings: compare compact size to what legacy size would be // For events that are in compact format, estimate legacy size based on typical ratios // A typical event has: // - 32 bytes event ID (saved in compact: stored separately in sei) // - 32 bytes pubkey (saved: replaced by 5-byte serial) // - For e-tags: 32 bytes each (saved: replaced by 5-byte serial when known) // - For p-tags: 32 bytes each (saved: replaced by 5-byte serial) // Conservative estimate: compact format is ~60% of legacy size for typical events if stats.CompactEvents > 0 && stats.AverageCompact > 0 { // Estimate what the legacy size would have been estimatedLegacyForCompact := float64(stats.CompactBytes) / 0.60 // 60% compression ratio stats.BytesSaved = int64(estimatedLegacyForCompact) - stats.CompactBytes - stats.SerialEventIdBytes if stats.BytesSaved < 0 { stats.BytesSaved = 0 } totalWithoutCompact := estimatedLegacyForCompact + float64(stats.LegacyBytes) totalWithCompact := float64(stats.CompactBytes + stats.LegacyBytes + stats.SerialEventIdBytes) if totalWithoutCompact > 0 { stats.PercentSaved = (1.0 - totalWithCompact/totalWithoutCompact) * 100.0 } } return stats, nil } // compactSaveCounter tracks cumulative bytes saved by compact format var compactSaveCounter atomic.Int64 // LogCompactSavings logs the storage savings achieved by compact format. // Call this periodically or after significant operations. func (d *D) LogCompactSavings() { stats, err := d.CompactStorageStats() if err != nil { log.W.F("failed to get compact storage stats: %v", err) return } if stats.TotalEvents == 0 { return } log.I.F("📊 Compact storage stats: %d compact events, %d legacy events", stats.CompactEvents, stats.LegacyEvents) log.I.F(" Compact size: %.2f MB, Legacy size: %.2f MB", float64(stats.CompactBytes)/(1024.0*1024.0), float64(stats.LegacyBytes)/(1024.0*1024.0)) log.I.F(" Serial mappings (sei): %d entries, %.2f KB", stats.SerialEventIdEntries, float64(stats.SerialEventIdBytes)/1024.0) if stats.CompactEvents > 0 { log.I.F(" Average compact event: %.0f bytes, estimated legacy: %.0f bytes", stats.AverageCompact, stats.AverageCompact/0.60) log.I.F(" Estimated savings: %.2f MB (%.1f%%)", float64(stats.BytesSaved)/(1024.0*1024.0), stats.PercentSaved) } // Also log serial cache stats cacheStats := d.SerialCacheStats() log.I.F(" Serial cache: %d/%d pubkeys, %d/%d event IDs, ~%.2f MB memory", cacheStats.PubkeysCached, cacheStats.PubkeysMaxSize, cacheStats.EventIdsCached, cacheStats.EventIdsMaxSize, float64(cacheStats.TotalMemoryBytes)/(1024.0*1024.0)) } // TrackCompactSaving records bytes saved for a single event. // Call this during event save to track cumulative savings. func TrackCompactSaving(legacySize, compactSize int) { saved := legacySize - compactSize if saved > 0 { compactSaveCounter.Add(int64(saved)) } } // GetCumulativeCompactSavings returns total bytes saved across all compact saves. func GetCumulativeCompactSavings() int64 { return compactSaveCounter.Load() } // ResetCompactSavingsCounter resets the cumulative savings counter. func ResetCompactSavingsCounter() { compactSaveCounter.Store(0) }