Interim release: documentation updates and rate limiting improvements
- Add applesauce library reference documentation - Add rate limiting test report for Badger - Add memory monitoring for rate limiter (platform-specific implementations) - Enhance PID-controlled adaptive rate limiting - Update Neo4j and Badger monitors with improved load metrics - Add docker-compose configuration - Update README and configuration options 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit was merged in pull request #3.
This commit is contained in:
@@ -106,18 +106,21 @@ type C struct {
|
||||
SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
|
||||
|
||||
// Adaptive rate limiting (PID-controlled)
|
||||
RateLimitEnabled bool `env:"ORLY_RATE_LIMIT_ENABLED" default:"false" usage:"enable adaptive PID-controlled rate limiting for database operations"`
|
||||
RateLimitTargetMB int `env:"ORLY_RATE_LIMIT_TARGET_MB" default:"1500" usage:"target memory limit in MB for rate limiting (default: 1500 = 1.5GB)"`
|
||||
RateLimitWriteKp float64 `env:"ORLY_RATE_LIMIT_WRITE_KP" default:"0.5" usage:"PID proportional gain for write operations"`
|
||||
RateLimitWriteKi float64 `env:"ORLY_RATE_LIMIT_WRITE_KI" default:"0.1" usage:"PID integral gain for write operations"`
|
||||
RateLimitWriteKd float64 `env:"ORLY_RATE_LIMIT_WRITE_KD" default:"0.05" usage:"PID derivative gain for write operations (filtered)"`
|
||||
RateLimitReadKp float64 `env:"ORLY_RATE_LIMIT_READ_KP" default:"0.3" usage:"PID proportional gain for read operations"`
|
||||
RateLimitReadKi float64 `env:"ORLY_RATE_LIMIT_READ_KI" default:"0.05" usage:"PID integral gain for read operations"`
|
||||
RateLimitReadKd float64 `env:"ORLY_RATE_LIMIT_READ_KD" default:"0.02" usage:"PID derivative gain for read operations (filtered)"`
|
||||
RateLimitMaxWriteMs int `env:"ORLY_RATE_LIMIT_MAX_WRITE_MS" default:"1000" usage:"maximum delay for write operations in milliseconds"`
|
||||
RateLimitMaxReadMs int `env:"ORLY_RATE_LIMIT_MAX_READ_MS" default:"500" usage:"maximum delay for read operations in milliseconds"`
|
||||
RateLimitWriteTarget float64 `env:"ORLY_RATE_LIMIT_WRITE_TARGET" default:"0.85" usage:"PID setpoint for writes (throttle when load exceeds this, 0.0-1.0)"`
|
||||
RateLimitReadTarget float64 `env:"ORLY_RATE_LIMIT_READ_TARGET" default:"0.90" usage:"PID setpoint for reads (throttle when load exceeds this, 0.0-1.0)"`
|
||||
RateLimitEnabled bool `env:"ORLY_RATE_LIMIT_ENABLED" default:"true" usage:"enable adaptive PID-controlled rate limiting for database operations"`
|
||||
RateLimitTargetMB int `env:"ORLY_RATE_LIMIT_TARGET_MB" default:"0" usage:"target memory limit in MB (0=auto-detect: 66% of available, min 500MB)"`
|
||||
RateLimitWriteKp float64 `env:"ORLY_RATE_LIMIT_WRITE_KP" default:"0.5" usage:"PID proportional gain for write operations"`
|
||||
RateLimitWriteKi float64 `env:"ORLY_RATE_LIMIT_WRITE_KI" default:"0.1" usage:"PID integral gain for write operations"`
|
||||
RateLimitWriteKd float64 `env:"ORLY_RATE_LIMIT_WRITE_KD" default:"0.05" usage:"PID derivative gain for write operations (filtered)"`
|
||||
RateLimitReadKp float64 `env:"ORLY_RATE_LIMIT_READ_KP" default:"0.3" usage:"PID proportional gain for read operations"`
|
||||
RateLimitReadKi float64 `env:"ORLY_RATE_LIMIT_READ_KI" default:"0.05" usage:"PID integral gain for read operations"`
|
||||
RateLimitReadKd float64 `env:"ORLY_RATE_LIMIT_READ_KD" default:"0.02" usage:"PID derivative gain for read operations (filtered)"`
|
||||
RateLimitMaxWriteMs int `env:"ORLY_RATE_LIMIT_MAX_WRITE_MS" default:"1000" usage:"maximum delay for write operations in milliseconds"`
|
||||
RateLimitMaxReadMs int `env:"ORLY_RATE_LIMIT_MAX_READ_MS" default:"500" usage:"maximum delay for read operations in milliseconds"`
|
||||
RateLimitWriteTarget float64 `env:"ORLY_RATE_LIMIT_WRITE_TARGET" default:"0.85" usage:"PID setpoint for writes (throttle when load exceeds this, 0.0-1.0)"`
|
||||
RateLimitReadTarget float64 `env:"ORLY_RATE_LIMIT_READ_TARGET" default:"0.90" usage:"PID setpoint for reads (throttle when load exceeds this, 0.0-1.0)"`
|
||||
RateLimitEmergencyThreshold float64 `env:"ORLY_RATE_LIMIT_EMERGENCY_THRESHOLD" default:"1.167" usage:"memory pressure ratio (target+1/6) to trigger emergency mode with aggressive throttling"`
|
||||
RateLimitRecoveryThreshold float64 `env:"ORLY_RATE_LIMIT_RECOVERY_THRESHOLD" default:"0.833" usage:"memory pressure ratio (target-1/6) below which emergency mode exits (hysteresis)"`
|
||||
RateLimitEmergencyMaxMs int `env:"ORLY_RATE_LIMIT_EMERGENCY_MAX_MS" default:"5000" usage:"maximum delay for writes in emergency mode (milliseconds)"`
|
||||
|
||||
// TLS configuration
|
||||
TLSDomains []string `env:"ORLY_TLS_DOMAINS" usage:"comma-separated list of domains to respond to for TLS"`
|
||||
@@ -457,11 +460,15 @@ func (cfg *C) GetRateLimitConfigValues() (
|
||||
readKp, readKi, readKd float64,
|
||||
maxWriteMs, maxReadMs int,
|
||||
writeTarget, readTarget float64,
|
||||
emergencyThreshold, recoveryThreshold float64,
|
||||
emergencyMaxMs int,
|
||||
) {
|
||||
return cfg.RateLimitEnabled,
|
||||
cfg.RateLimitTargetMB,
|
||||
cfg.RateLimitWriteKp, cfg.RateLimitWriteKi, cfg.RateLimitWriteKd,
|
||||
cfg.RateLimitReadKp, cfg.RateLimitReadKi, cfg.RateLimitReadKd,
|
||||
cfg.RateLimitMaxWriteMs, cfg.RateLimitMaxReadMs,
|
||||
cfg.RateLimitWriteTarget, cfg.RateLimitReadTarget
|
||||
cfg.RateLimitWriteTarget, cfg.RateLimitReadTarget,
|
||||
cfg.RateLimitEmergencyThreshold, cfg.RateLimitRecoveryThreshold,
|
||||
cfg.RateLimitEmergencyMaxMs
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"git.mleku.dev/mleku/nostr/encoders/kind"
|
||||
"git.mleku.dev/mleku/nostr/encoders/reason"
|
||||
"next.orly.dev/pkg/protocol/nip43"
|
||||
"next.orly.dev/pkg/ratelimit"
|
||||
"next.orly.dev/pkg/utils"
|
||||
)
|
||||
|
||||
@@ -608,6 +609,10 @@ func (l *Listener) HandleEvent(msg []byte) (err error) {
|
||||
env.E.Pubkey,
|
||||
)
|
||||
log.I.F("delete event pubkey hex: %s", hex.Enc(env.E.Pubkey))
|
||||
// Apply rate limiting before write operation
|
||||
if l.rateLimiter != nil && l.rateLimiter.IsEnabled() {
|
||||
l.rateLimiter.Wait(saveCtx, int(ratelimit.Write))
|
||||
}
|
||||
if _, err = l.DB.SaveEvent(saveCtx, env.E); err != nil {
|
||||
log.E.F("failed to save delete event %0x: %v", env.E.ID, err)
|
||||
if strings.HasPrefix(err.Error(), "blocked:") {
|
||||
@@ -675,6 +680,10 @@ func (l *Listener) HandleEvent(msg []byte) (err error) {
|
||||
// store the event - use a separate context to prevent cancellation issues
|
||||
saveCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
// Apply rate limiting before write operation
|
||||
if l.rateLimiter != nil && l.rateLimiter.IsEnabled() {
|
||||
l.rateLimiter.Wait(saveCtx, int(ratelimit.Write))
|
||||
}
|
||||
// log.I.F("saving event %0x, %s", env.E.ID, env.E.Serialize())
|
||||
if _, err = l.DB.SaveEvent(saveCtx, env.E); err != nil {
|
||||
if strings.HasPrefix(err.Error(), "blocked:") {
|
||||
|
||||
Reference in New Issue
Block a user