Add archive relay query augmentation and access-based GC (v0.45.0)

- Add async archive relay querying (local results immediate, archives in background)
- Add query caching with filter normalization to avoid repeated requests
- Add session-deduplicated access tracking for events
- Add continuous garbage collection based on access patterns
- Auto-detect storage limit (80% of filesystem) when ORLY_MAX_STORAGE_BYTES=0
- Support NIP-50 search queries to archive relays

New environment variables:
- ORLY_ARCHIVE_ENABLED: Enable archive relay query augmentation
- ORLY_ARCHIVE_RELAYS: Comma-separated archive relay URLs
- ORLY_ARCHIVE_TIMEOUT_SEC: Archive query timeout
- ORLY_ARCHIVE_CACHE_TTL_HRS: Query deduplication window
- ORLY_GC_ENABLED: Enable access-based garbage collection
- ORLY_MAX_STORAGE_BYTES: Max storage (0=auto 80%)
- ORLY_GC_INTERVAL_SEC: GC check interval
- ORLY_GC_BATCH_SIZE: Events per GC cycle

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

app/config/config.go

@@ -160,6 +160,18 @@ type C struct {
 	// Cluster replication configuration
 	ClusterPropagatePrivilegedEvents bool `env:"ORLY_CLUSTER_PROPAGATE_PRIVILEGED_EVENTS" default:"true" usage:"propagate privileged events (DMs, gift wraps, etc.) to relay peers for replication"`
 
+	// Archive relay configuration (query augmentation from authoritative archives)
+	ArchiveEnabled     bool     `env:"ORLY_ARCHIVE_ENABLED" default:"false" usage:"enable archive relay query augmentation (fetch from archives, cache locally)"`
+	ArchiveRelays      []string `env:"ORLY_ARCHIVE_RELAYS" default:"wss://archive.orly.dev/" usage:"comma-separated list of archive relay URLs for query augmentation"`
+	ArchiveTimeoutSec  int      `env:"ORLY_ARCHIVE_TIMEOUT_SEC" default:"30" usage:"timeout in seconds for archive relay queries"`
+	ArchiveCacheTTLHrs int      `env:"ORLY_ARCHIVE_CACHE_TTL_HRS" default:"24" usage:"hours to cache query fingerprints to avoid repeated archive requests"`
+
+	// Storage management configuration (access-based garbage collection)
+	MaxStorageBytes int64 `env:"ORLY_MAX_STORAGE_BYTES" default:"0" usage:"maximum storage in bytes (0=auto-detect 80%% of filesystem)"`
+	GCEnabled       bool  `env:"ORLY_GC_ENABLED" default:"true" usage:"enable continuous garbage collection based on access patterns"`
+	GCIntervalSec   int   `env:"ORLY_GC_INTERVAL_SEC" default:"60" usage:"seconds between GC runs when storage exceeds limit"`
+	GCBatchSize     int   `env:"ORLY_GC_BATCH_SIZE" default:"1000" usage:"number of events to consider per GC run"`
+
 	// ServeMode is set programmatically by the 'serve' subcommand to grant full owner
 	// access to all users (no env tag - internal use only)
 	ServeMode bool
@@ -590,3 +602,33 @@ func (cfg *C) GetCashuConfigValues() (
 		scopes,
 		cfg.CashuReauthorize
 }
+
+// GetArchiveConfigValues returns the archive relay configuration values.
+// This avoids circular imports with pkg/archive while allowing main.go to construct
+// the archive manager configuration.
+func (cfg *C) GetArchiveConfigValues() (
+	enabled bool,
+	relays []string,
+	timeoutSec int,
+	cacheTTLHrs int,
+) {
+	return cfg.ArchiveEnabled,
+		cfg.ArchiveRelays,
+		cfg.ArchiveTimeoutSec,
+		cfg.ArchiveCacheTTLHrs
+}
+
+// GetStorageConfigValues returns the storage management configuration values.
+// This avoids circular imports with pkg/storage while allowing main.go to construct
+// the garbage collector and access tracker configuration.
+func (cfg *C) GetStorageConfigValues() (
+	maxStorageBytes int64,
+	gcEnabled bool,
+	gcIntervalSec int,
+	gcBatchSize int,
+) {
+	return cfg.MaxStorageBytes,
+		cfg.GCEnabled,
+		cfg.GCIntervalSec,
+		cfg.GCBatchSize
+}

app/handle-req.go

@@ -690,6 +690,31 @@ func (l *Listener) HandleReq(msg []byte) (err error) {
 		Write(l); chk.E(err) {
 		return
 	}
+
+	// Record access for returned events (for GC access-based ranking)
+	if l.accessTracker != nil && len(events) > 0 {
+		go func(evts event.S, connID string) {
+			for _, ev := range evts {
+				if ser, err := l.DB.GetSerialById(ev.ID); err == nil && ser != nil {
+					l.accessTracker.RecordAccess(ser.Get(), connID)
+				}
+			}
+		}(events, l.connectionID)
+	}
+
+	// Trigger archive relay query if enabled (background fetch + stream results)
+	if l.archiveManager != nil && l.archiveManager.IsEnabled() && len(*env.Filters) > 0 {
+		// Use first filter for archive query
+		f := (*env.Filters)[0]
+		go l.archiveManager.QueryArchive(
+			string(env.Subscription),
+			l.connectionID,
+			f,
+			seen,
+			l, // implements EventDeliveryChannel
+		)
+	}
+
 	// if the query was for just Ids, we know there can't be any more results,
 	// so cancel the subscription.
 	cancel := true

app/handle-websocket.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"crypto/rand"
+	"fmt"
 	"net/http"
 	"strings"
 	"time"
@@ -99,15 +100,17 @@ whitelist:
 		handlerSemSize = 100 // Default if not configured
 	}
 
+	now := time.Now()
 	listener := &Listener{
 		ctx:            ctx,
 		cancel:         cancel,
 		Server:         s,
 		conn:           conn,
 		remote:         remote,
+		connectionID:   fmt.Sprintf("%s-%d", remote, now.UnixNano()), // Unique connection ID for access tracking
 		req:            r,
 		cashuToken:     cashuToken, // Verified Cashu access token (nil if none provided)
-		startTime:      time.Now(),
+		startTime:      now,
 		writeChan:      make(chan publish.WriteRequest, 100), // Buffered channel for writes
 		writeDone:      make(chan struct{}),
 		messageQueue:   make(chan messageRequest, 100), // Buffered channel for message processing

app/listener.go

@@ -28,6 +28,7 @@ type Listener struct {
 	ctx              context.Context
 	cancel           context.CancelFunc // Cancel function for this listener's context
 	remote           string
+	connectionID     string // Unique identifier for this connection (for access tracking)
 	req              *http.Request
 	challenge        atomicutils.Bytes
 	authedPubkey     atomicutils.Bytes
@@ -112,6 +113,29 @@ func (l *Listener) Write(p []byte) (n int, err error) {
 	}
 }
 
+// SendEvent sends an event to the client. Implements archive.EventDeliveryChannel.
+func (l *Listener) SendEvent(ev *event.E) error {
+	if ev == nil {
+		return nil
+	}
+	// Serialize the event as an EVENT envelope
+	data := ev.Serialize()
+	// Use Write to send
+	_, err := l.Write(data)
+	return err
+}
+
+// IsConnected returns whether the client connection is still active.
+// Implements archive.EventDeliveryChannel.
+func (l *Listener) IsConnected() bool {
+	select {
+	case <-l.ctx.Done():
+		return false
+	default:
+		return l.conn != nil
+	}
+}
+
 // WriteControl sends a control message through the write channel
 func (l *Listener) WriteControl(messageType int, data []byte, deadline time.Time) (err error) {
 	// Defensive: recover from any panic when sending to closed channel

app/main.go

@@ -29,8 +29,10 @@ import (
 	cashuiface "next.orly.dev/pkg/interfaces/cashu"
 	"next.orly.dev/pkg/ratelimit"
 	"next.orly.dev/pkg/spider"
+	"next.orly.dev/pkg/storage"
 	dsync "next.orly.dev/pkg/sync"
 	"next.orly.dev/pkg/wireguard"
+	"next.orly.dev/pkg/archive"
 
 	"git.mleku.dev/mleku/nostr/interfaces/signer/p8k"
 )
@@ -512,6 +514,40 @@ func Run(
 		}
 	}
 
+	// Initialize access tracker for storage management (only for Badger backend)
+	if badgerDB, ok := db.(*database.D); ok {
+		l.accessTracker = storage.NewAccessTracker(badgerDB, 100000) // 100k dedup cache
+		l.accessTracker.Start()
+		log.I.F("access tracker initialized")
+
+		// Initialize garbage collector if enabled
+		maxBytes, gcEnabled, gcIntervalSec, gcBatchSize := cfg.GetStorageConfigValues()
+		if gcEnabled {
+			gcCfg := storage.GCConfig{
+				MaxStorageBytes: maxBytes,
+				Interval:        time.Duration(gcIntervalSec) * time.Second,
+				BatchSize:       gcBatchSize,
+				MinAgeSec:       3600, // Minimum 1 hour before eviction
+			}
+			l.garbageCollector = storage.NewGarbageCollector(ctx, badgerDB, l.accessTracker, gcCfg)
+			l.garbageCollector.Start()
+			log.I.F("garbage collector started (interval: %ds, batch: %d)", gcIntervalSec, gcBatchSize)
+		}
+	}
+
+	// Initialize archive relay manager if enabled
+	archiveEnabled, archiveRelays, archiveTimeoutSec, archiveCacheTTLHrs := cfg.GetArchiveConfigValues()
+	if archiveEnabled && len(archiveRelays) > 0 {
+		archiveCfg := archive.Config{
+			Enabled:     true,
+			Relays:      archiveRelays,
+			TimeoutSec:  archiveTimeoutSec,
+			CacheTTLHrs: archiveCacheTTLHrs,
+		}
+		l.archiveManager = archive.New(ctx, db, archiveCfg)
+		log.I.F("archive relay manager initialized with %d relays", len(archiveRelays))
+	}
+
 	// Start rate limiter if enabled
 	if limiter != nil && limiter.IsEnabled() {
 		limiter.Start()
@@ -621,6 +657,24 @@ func Run(
 			log.I.F("rate limiter stopped")
 		}
 
+		// Stop archive manager if running
+		if l.archiveManager != nil {
+			l.archiveManager.Stop()
+			log.I.F("archive manager stopped")
+		}
+
+		// Stop garbage collector if running
+		if l.garbageCollector != nil {
+			l.garbageCollector.Stop()
+			log.I.F("garbage collector stopped")
+		}
+
+		// Stop access tracker if running
+		if l.accessTracker != nil {
+			l.accessTracker.Stop()
+			log.I.F("access tracker stopped")
+		}
+
 		// Stop bunker server if running
 		if l.bunkerServer != nil {
 			l.bunkerServer.Stop()

app/server.go

@@ -38,8 +38,10 @@ import (
 	"next.orly.dev/pkg/cashu/verifier"
 	"next.orly.dev/pkg/ratelimit"
 	"next.orly.dev/pkg/spider"
+	"next.orly.dev/pkg/storage"
 	dsync "next.orly.dev/pkg/sync"
 	"next.orly.dev/pkg/wireguard"
+	"next.orly.dev/pkg/archive"
 )
 
 type Server struct {
@@ -91,6 +93,11 @@ type Server struct {
 	// Cashu access token system (NIP-XX)
 	CashuIssuer   *issuer.Issuer
 	CashuVerifier *verifier.Verifier
+
+	// Archive relay and storage management
+	archiveManager   *archive.Manager
+	accessTracker    *storage.AccessTracker
+	garbageCollector *storage.GarbageCollector
 }
 
 // isIPBlacklisted checks if an IP address is blacklisted using the managed ACL system