Implement PID-controlled adaptive rate limiting for database operations

- Add LoadMonitor interface in pkg/interfaces/loadmonitor/ for database load metrics
- Implement PIDController with filtered derivative to suppress high-frequency noise
  - Proportional (P): immediate response to current error
  - Integral (I): eliminates steady-state offset with anti-windup clamping
  - Derivative (D): rate-of-change prediction with low-pass filtering
- Create BadgerLoadMonitor tracking L0 tables, compaction score, and cache hit ratio
- Create Neo4jLoadMonitor tracking query semaphore usage and latencies
- Add AdaptiveRateLimiter combining PID controllers for reads and writes
- Configure via environment variables:
  - ORLY_RATE_LIMIT_ENABLED: enable/disable rate limiting
  - ORLY_RATE_LIMIT_TARGET_MB: target memory limit (default 1500MB)
  - ORLY_RATE_LIMIT_*_K[PID]: PID gains for reads/writes
  - ORLY_RATE_LIMIT_MAX_*_MS: maximum delays
  - ORLY_RATE_LIMIT_*_TARGET: setpoints for reads/writes
- Integrate rate limiter into Server struct and lifecycle management
- Add comprehensive unit tests for PID controller behavior

Files modified:
- app/config/config.go: Add rate limiting configuration options
- app/main.go: Initialize and start/stop rate limiter
- app/server.go: Add rateLimiter field to Server struct
- main.go: Create rate limiter with appropriate monitor
- pkg/run/run.go: Pass disabled limiter for test instances
- pkg/interfaces/loadmonitor/: New LoadMonitor interface
- pkg/ratelimit/: New PID controller and limiter implementation

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

app/config/config.go

@@ -102,8 +102,22 @@ type C struct {
 	Neo4jPassword string `env:"ORLY_NEO4J_PASSWORD" default:"password" usage:"Neo4j authentication password (only used when ORLY_DB_TYPE=neo4j)"`
 
 	// Advanced database tuning
-	SerialCachePubkeys     int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"`
-	SerialCacheEventIds    int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
+	SerialCachePubkeys  int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"`
+	SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
+
+	// Adaptive rate limiting (PID-controlled)
+	RateLimitEnabled     bool    `env:"ORLY_RATE_LIMIT_ENABLED" default:"false" usage:"enable adaptive PID-controlled rate limiting for database operations"`
+	RateLimitTargetMB    int     `env:"ORLY_RATE_LIMIT_TARGET_MB" default:"1500" usage:"target memory limit in MB for rate limiting (default: 1500 = 1.5GB)"`
+	RateLimitWriteKp     float64 `env:"ORLY_RATE_LIMIT_WRITE_KP" default:"0.5" usage:"PID proportional gain for write operations"`
+	RateLimitWriteKi     float64 `env:"ORLY_RATE_LIMIT_WRITE_KI" default:"0.1" usage:"PID integral gain for write operations"`
+	RateLimitWriteKd     float64 `env:"ORLY_RATE_LIMIT_WRITE_KD" default:"0.05" usage:"PID derivative gain for write operations (filtered)"`
+	RateLimitReadKp      float64 `env:"ORLY_RATE_LIMIT_READ_KP" default:"0.3" usage:"PID proportional gain for read operations"`
+	RateLimitReadKi      float64 `env:"ORLY_RATE_LIMIT_READ_KI" default:"0.05" usage:"PID integral gain for read operations"`
+	RateLimitReadKd      float64 `env:"ORLY_RATE_LIMIT_READ_KD" default:"0.02" usage:"PID derivative gain for read operations (filtered)"`
+	RateLimitMaxWriteMs  int     `env:"ORLY_RATE_LIMIT_MAX_WRITE_MS" default:"1000" usage:"maximum delay for write operations in milliseconds"`
+	RateLimitMaxReadMs   int     `env:"ORLY_RATE_LIMIT_MAX_READ_MS" default:"500" usage:"maximum delay for read operations in milliseconds"`
+	RateLimitWriteTarget float64 `env:"ORLY_RATE_LIMIT_WRITE_TARGET" default:"0.85" usage:"PID setpoint for writes (throttle when load exceeds this, 0.0-1.0)"`
+	RateLimitReadTarget  float64 `env:"ORLY_RATE_LIMIT_READ_TARGET" default:"0.90" usage:"PID setpoint for reads (throttle when load exceeds this, 0.0-1.0)"`
 
 	// TLS configuration
 	TLSDomains []string `env:"ORLY_TLS_DOMAINS" usage:"comma-separated list of domains to respond to for TLS"`
@@ -432,3 +446,22 @@ func (cfg *C) GetDatabaseConfigValues() (
 		cfg.DBZSTDLevel,
 		cfg.Neo4jURI, cfg.Neo4jUser, cfg.Neo4jPassword
 }
+
+// GetRateLimitConfigValues returns the rate limiting configuration values.
+// This avoids circular imports with pkg/ratelimit while allowing main.go to construct
+// a ratelimit.Config with the correct type.
+func (cfg *C) GetRateLimitConfigValues() (
+	enabled bool,
+	targetMB int,
+	writeKp, writeKi, writeKd float64,
+	readKp, readKi, readKd float64,
+	maxWriteMs, maxReadMs int,
+	writeTarget, readTarget float64,
+) {
+	return cfg.RateLimitEnabled,
+		cfg.RateLimitTargetMB,
+		cfg.RateLimitWriteKp, cfg.RateLimitWriteKi, cfg.RateLimitWriteKd,
+		cfg.RateLimitReadKp, cfg.RateLimitReadKi, cfg.RateLimitReadKd,
+		cfg.RateLimitMaxWriteMs, cfg.RateLimitMaxReadMs,
+		cfg.RateLimitWriteTarget, cfg.RateLimitReadTarget
+}

app/main.go

@@ -21,12 +21,13 @@ import (
 	"next.orly.dev/pkg/protocol/graph"
 	"next.orly.dev/pkg/protocol/nip43"
 	"next.orly.dev/pkg/protocol/publish"
+	"next.orly.dev/pkg/ratelimit"
 	"next.orly.dev/pkg/spider"
 	dsync "next.orly.dev/pkg/sync"
 )
 
 func Run(
-	ctx context.Context, cfg *config.C, db database.Database,
+	ctx context.Context, cfg *config.C, db database.Database, limiter *ratelimit.Limiter,
 ) (quit chan struct{}) {
 	quit = make(chan struct{})
 	var once sync.Once
@@ -64,14 +65,15 @@ func Run(
 	}
 	// start listener
 	l := &Server{
-		Ctx:        ctx,
-		Config:     cfg,
-		DB:         db,
-		publishers: publish.New(NewPublisher(ctx)),
-		Admins:     adminKeys,
-		Owners:     ownerKeys,
-		cfg:        cfg,
-		db:         db,
+		Ctx:         ctx,
+		Config:      cfg,
+		DB:          db,
+		publishers:  publish.New(NewPublisher(ctx)),
+		Admins:      adminKeys,
+		Owners:      ownerKeys,
+		rateLimiter: limiter,
+		cfg:         cfg,
+		db:          db,
 	}
 
 	// Initialize NIP-43 invite manager if enabled
@@ -360,6 +362,12 @@ func Run(
 		}
 	}
 
+	// Start rate limiter if enabled
+	if limiter != nil && limiter.IsEnabled() {
+		limiter.Start()
+		log.I.F("adaptive rate limiter started")
+	}
+
 	// Wait for database to be ready before accepting requests
 	log.I.F("waiting for database warmup to complete...")
 	<-db.Ready()
@@ -457,6 +465,12 @@ func Run(
 			log.I.F("directory spider stopped")
 		}
 
+		// Stop rate limiter if running
+		if l.rateLimiter != nil && l.rateLimiter.IsEnabled() {
+			l.rateLimiter.Stop()
+			log.I.F("rate limiter stopped")
+		}
+
 		// Create shutdown context with timeout
 		shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancelShutdown()

app/server.go

@@ -29,6 +29,7 @@ import (
 	"next.orly.dev/pkg/protocol/graph"
 	"next.orly.dev/pkg/protocol/nip43"
 	"next.orly.dev/pkg/protocol/publish"
+	"next.orly.dev/pkg/ratelimit"
 	"next.orly.dev/pkg/spider"
 	dsync "next.orly.dev/pkg/sync"
 )
@@ -64,6 +65,7 @@ type Server struct {
 	blossomServer     *blossom.Server
 	InviteManager     *nip43.InviteManager
 	graphExecutor     *graph.Executor
+	rateLimiter       *ratelimit.Limiter
 	cfg               *config.C
 	db                database.Database // Changed from *database.D to interface
 }