Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
a816737cd3
|
|||
|
28b41847a6
|
|||
|
88b0509ad8
|
|||
|
afa3dce1c9
|
|||
|
cbc502a703
|
@@ -1,4 +1,5 @@
|
||||
{
|
||||
"MAX_THINKING_TOKENS": "8000",
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash:*",
|
||||
@@ -57,7 +58,34 @@
|
||||
"Bash(go clean:*)",
|
||||
"Bash(go mod tidy:*)",
|
||||
"Bash(./scripts/test-neo4j-integration.sh:*)",
|
||||
"Bash(docker compose:*)"
|
||||
"Bash(docker compose:*)",
|
||||
"Bash(sudo update-grub:*)",
|
||||
"Bash(lspci:*)",
|
||||
"Bash(lsmod:*)",
|
||||
"Bash(modinfo:*)",
|
||||
"Bash(apt-cache policy:*)",
|
||||
"WebFetch(domain:git.kernel.org)",
|
||||
"Bash(ip link:*)",
|
||||
"WebFetch(domain:www.laptopcentar.rs)",
|
||||
"WebFetch(domain:www.kupujemprodajem.com)",
|
||||
"WebFetch(domain:www.bcgroup-online.com)",
|
||||
"WebFetch(domain:www.monitor.rs)",
|
||||
"WebFetch(domain:www.protis.hr)",
|
||||
"Bash(apt-cache search:*)",
|
||||
"Bash(dkms status:*)",
|
||||
"Bash(sudo dkms build:*)",
|
||||
"Bash(sudo apt install:*)",
|
||||
"Bash(wget:*)",
|
||||
"Bash(ls:*)",
|
||||
"Bash(git clone:*)",
|
||||
"Bash(sudo make:*)",
|
||||
"Bash(sudo modprobe:*)",
|
||||
"Bash(update-desktop-database:*)",
|
||||
"Bash(CGO_ENABLED=0 go build:*)",
|
||||
"Bash(CGO_ENABLED=0 go test:*)",
|
||||
"Bash(git submodule:*)",
|
||||
"WebFetch(domain:neo4j.com)",
|
||||
"Bash(git reset:*)"
|
||||
],
|
||||
"deny": [],
|
||||
"ask": []
|
||||
|
||||
@@ -150,10 +150,20 @@ Event kind `7` for reactions:
|
||||
|
||||
#### NIP-42: Authentication
|
||||
Client authentication to relays:
|
||||
- AUTH message from relay
|
||||
- Client responds with event kind `22242`
|
||||
- AUTH message from relay (challenge)
|
||||
- Client responds with event kind `22242` signed auth event
|
||||
- Proves key ownership
|
||||
|
||||
**CRITICAL: Clients MUST wait for OK response after AUTH**
|
||||
- Relays MUST respond to AUTH with an OK message (same as EVENT)
|
||||
- An OK with `true` confirms the relay has stored the authenticated pubkey
|
||||
- An OK with `false` indicates authentication failed:
|
||||
1. **Alert the user** that authentication failed
|
||||
2. **Assume the relay will reject** subsequent events requiring auth
|
||||
3. Check the `reason` field for error details (e.g., "error: failed to parse auth event")
|
||||
- Do NOT send events requiring authentication until OK `true` is received
|
||||
- If no OK is received within timeout, assume connection issues and retry or alert user
|
||||
|
||||
#### NIP-50: Search
|
||||
Query filter extension for full-text search:
|
||||
- `search` field in REQ filters
|
||||
|
||||
12
CLAUDE.md
12
CLAUDE.md
@@ -901,6 +901,18 @@ WebAssembly-compatible database backend (`pkg/wasmdb/`):
|
||||
- `ORLY_AUTH_REQUIRED=true`: Require authentication for ALL requests
|
||||
- `ORLY_AUTH_TO_WRITE=true`: Require authentication only for writes (allow anonymous reads)
|
||||
|
||||
### NIP-42 AUTH Protocol (IMPORTANT for Client Developers)
|
||||
Per NIP-42, this relay always responds to AUTH messages with an OK message:
|
||||
- **Clients MUST wait for the OK response** after sending AUTH before publishing events
|
||||
- An OK with `true` confirms the relay has stored the authenticated pubkey
|
||||
- An OK with `false` indicates authentication failed - clients should:
|
||||
1. Alert the user that authentication failed
|
||||
2. Assume the relay will reject subsequent events requiring auth
|
||||
3. Check the reason field for error details
|
||||
- If no OK is received within a reasonable timeout, assume connection issues
|
||||
|
||||
Implementation: `app/handle-auth.go`
|
||||
|
||||
### NIP-43 Relay Access Metadata
|
||||
Invite-based access control system:
|
||||
- `ORLY_NIP43_ENABLED=true`: Enable invite system
|
||||
|
||||
@@ -102,8 +102,22 @@ type C struct {
|
||||
Neo4jPassword string `env:"ORLY_NEO4J_PASSWORD" default:"password" usage:"Neo4j authentication password (only used when ORLY_DB_TYPE=neo4j)"`
|
||||
|
||||
// Advanced database tuning
|
||||
SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"`
|
||||
SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
|
||||
SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"max pubkeys to cache for compact event storage (default: 100000, ~3.2MB memory)"`
|
||||
SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"max event IDs to cache for compact event storage (default: 500000, ~16MB memory)"`
|
||||
|
||||
// Adaptive rate limiting (PID-controlled)
|
||||
RateLimitEnabled bool `env:"ORLY_RATE_LIMIT_ENABLED" default:"false" usage:"enable adaptive PID-controlled rate limiting for database operations"`
|
||||
RateLimitTargetMB int `env:"ORLY_RATE_LIMIT_TARGET_MB" default:"1500" usage:"target memory limit in MB for rate limiting (default: 1500 = 1.5GB)"`
|
||||
RateLimitWriteKp float64 `env:"ORLY_RATE_LIMIT_WRITE_KP" default:"0.5" usage:"PID proportional gain for write operations"`
|
||||
RateLimitWriteKi float64 `env:"ORLY_RATE_LIMIT_WRITE_KI" default:"0.1" usage:"PID integral gain for write operations"`
|
||||
RateLimitWriteKd float64 `env:"ORLY_RATE_LIMIT_WRITE_KD" default:"0.05" usage:"PID derivative gain for write operations (filtered)"`
|
||||
RateLimitReadKp float64 `env:"ORLY_RATE_LIMIT_READ_KP" default:"0.3" usage:"PID proportional gain for read operations"`
|
||||
RateLimitReadKi float64 `env:"ORLY_RATE_LIMIT_READ_KI" default:"0.05" usage:"PID integral gain for read operations"`
|
||||
RateLimitReadKd float64 `env:"ORLY_RATE_LIMIT_READ_KD" default:"0.02" usage:"PID derivative gain for read operations (filtered)"`
|
||||
RateLimitMaxWriteMs int `env:"ORLY_RATE_LIMIT_MAX_WRITE_MS" default:"1000" usage:"maximum delay for write operations in milliseconds"`
|
||||
RateLimitMaxReadMs int `env:"ORLY_RATE_LIMIT_MAX_READ_MS" default:"500" usage:"maximum delay for read operations in milliseconds"`
|
||||
RateLimitWriteTarget float64 `env:"ORLY_RATE_LIMIT_WRITE_TARGET" default:"0.85" usage:"PID setpoint for writes (throttle when load exceeds this, 0.0-1.0)"`
|
||||
RateLimitReadTarget float64 `env:"ORLY_RATE_LIMIT_READ_TARGET" default:"0.90" usage:"PID setpoint for reads (throttle when load exceeds this, 0.0-1.0)"`
|
||||
|
||||
// TLS configuration
|
||||
TLSDomains []string `env:"ORLY_TLS_DOMAINS" usage:"comma-separated list of domains to respond to for TLS"`
|
||||
@@ -432,3 +446,22 @@ func (cfg *C) GetDatabaseConfigValues() (
|
||||
cfg.DBZSTDLevel,
|
||||
cfg.Neo4jURI, cfg.Neo4jUser, cfg.Neo4jPassword
|
||||
}
|
||||
|
||||
// GetRateLimitConfigValues returns the rate limiting configuration values.
|
||||
// This avoids circular imports with pkg/ratelimit while allowing main.go to construct
|
||||
// a ratelimit.Config with the correct type.
|
||||
func (cfg *C) GetRateLimitConfigValues() (
|
||||
enabled bool,
|
||||
targetMB int,
|
||||
writeKp, writeKi, writeKd float64,
|
||||
readKp, readKi, readKd float64,
|
||||
maxWriteMs, maxReadMs int,
|
||||
writeTarget, readTarget float64,
|
||||
) {
|
||||
return cfg.RateLimitEnabled,
|
||||
cfg.RateLimitTargetMB,
|
||||
cfg.RateLimitWriteKp, cfg.RateLimitWriteKi, cfg.RateLimitWriteKd,
|
||||
cfg.RateLimitReadKp, cfg.RateLimitReadKi, cfg.RateLimitReadKd,
|
||||
cfg.RateLimitMaxWriteMs, cfg.RateLimitMaxReadMs,
|
||||
cfg.RateLimitWriteTarget, cfg.RateLimitReadTarget
|
||||
}
|
||||
|
||||
@@ -5,13 +5,25 @@ import (
|
||||
"lol.mleku.dev/log"
|
||||
"git.mleku.dev/mleku/nostr/encoders/envelopes/authenvelope"
|
||||
"git.mleku.dev/mleku/nostr/encoders/envelopes/okenvelope"
|
||||
"git.mleku.dev/mleku/nostr/encoders/reason"
|
||||
"git.mleku.dev/mleku/nostr/protocol/auth"
|
||||
)
|
||||
|
||||
// zeroEventID is used for OK responses when we cannot parse the event ID
|
||||
var zeroEventID = make([]byte, 32)
|
||||
|
||||
func (l *Listener) HandleAuth(b []byte) (err error) {
|
||||
var rem []byte
|
||||
env := authenvelope.NewResponse()
|
||||
if rem, err = env.Unmarshal(b); chk.E(err) {
|
||||
// NIP-42: AUTH messages MUST be answered with an OK message
|
||||
// For parse failures, use zero event ID
|
||||
log.E.F("%s AUTH unmarshal failed: %v", l.remote, err)
|
||||
if writeErr := okenvelope.NewFrom(
|
||||
zeroEventID, false, reason.Error.F("failed to parse auth event: %s", err),
|
||||
).Write(l); chk.E(writeErr) {
|
||||
return writeErr
|
||||
}
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
|
||||
32
app/main.go
32
app/main.go
@@ -21,12 +21,13 @@ import (
|
||||
"next.orly.dev/pkg/protocol/graph"
|
||||
"next.orly.dev/pkg/protocol/nip43"
|
||||
"next.orly.dev/pkg/protocol/publish"
|
||||
"next.orly.dev/pkg/ratelimit"
|
||||
"next.orly.dev/pkg/spider"
|
||||
dsync "next.orly.dev/pkg/sync"
|
||||
)
|
||||
|
||||
func Run(
|
||||
ctx context.Context, cfg *config.C, db database.Database,
|
||||
ctx context.Context, cfg *config.C, db database.Database, limiter *ratelimit.Limiter,
|
||||
) (quit chan struct{}) {
|
||||
quit = make(chan struct{})
|
||||
var once sync.Once
|
||||
@@ -64,14 +65,15 @@ func Run(
|
||||
}
|
||||
// start listener
|
||||
l := &Server{
|
||||
Ctx: ctx,
|
||||
Config: cfg,
|
||||
DB: db,
|
||||
publishers: publish.New(NewPublisher(ctx)),
|
||||
Admins: adminKeys,
|
||||
Owners: ownerKeys,
|
||||
cfg: cfg,
|
||||
db: db,
|
||||
Ctx: ctx,
|
||||
Config: cfg,
|
||||
DB: db,
|
||||
publishers: publish.New(NewPublisher(ctx)),
|
||||
Admins: adminKeys,
|
||||
Owners: ownerKeys,
|
||||
rateLimiter: limiter,
|
||||
cfg: cfg,
|
||||
db: db,
|
||||
}
|
||||
|
||||
// Initialize NIP-43 invite manager if enabled
|
||||
@@ -360,6 +362,12 @@ func Run(
|
||||
}
|
||||
}
|
||||
|
||||
// Start rate limiter if enabled
|
||||
if limiter != nil && limiter.IsEnabled() {
|
||||
limiter.Start()
|
||||
log.I.F("adaptive rate limiter started")
|
||||
}
|
||||
|
||||
// Wait for database to be ready before accepting requests
|
||||
log.I.F("waiting for database warmup to complete...")
|
||||
<-db.Ready()
|
||||
@@ -457,6 +465,12 @@ func Run(
|
||||
log.I.F("directory spider stopped")
|
||||
}
|
||||
|
||||
// Stop rate limiter if running
|
||||
if l.rateLimiter != nil && l.rateLimiter.IsEnabled() {
|
||||
l.rateLimiter.Stop()
|
||||
log.I.F("rate limiter stopped")
|
||||
}
|
||||
|
||||
// Create shutdown context with timeout
|
||||
shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancelShutdown()
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"next.orly.dev/pkg/protocol/graph"
|
||||
"next.orly.dev/pkg/protocol/nip43"
|
||||
"next.orly.dev/pkg/protocol/publish"
|
||||
"next.orly.dev/pkg/ratelimit"
|
||||
"next.orly.dev/pkg/spider"
|
||||
dsync "next.orly.dev/pkg/sync"
|
||||
)
|
||||
@@ -64,6 +65,7 @@ type Server struct {
|
||||
blossomServer *blossom.Server
|
||||
InviteManager *nip43.InviteManager
|
||||
graphExecutor *graph.Executor
|
||||
rateLimiter *ratelimit.Limiter
|
||||
cfg *config.C
|
||||
db database.Database // Changed from *database.D to interface
|
||||
}
|
||||
|
||||
1250
docs/ADAPTIVE_RATE_LIMITING_PLAN.md
Normal file
1250
docs/ADAPTIVE_RATE_LIMITING_PLAN.md
Normal file
File diff suppressed because it is too large
Load Diff
34
main.go
34
main.go
@@ -23,6 +23,7 @@ import (
|
||||
"next.orly.dev/pkg/database"
|
||||
_ "next.orly.dev/pkg/neo4j" // Import to register neo4j factory
|
||||
"git.mleku.dev/mleku/nostr/encoders/hex"
|
||||
"next.orly.dev/pkg/ratelimit"
|
||||
"next.orly.dev/pkg/utils/interrupt"
|
||||
"next.orly.dev/pkg/version"
|
||||
)
|
||||
@@ -336,6 +337,37 @@ func main() {
|
||||
}
|
||||
acl.Registry.Syncer()
|
||||
|
||||
// Create rate limiter if enabled
|
||||
var limiter *ratelimit.Limiter
|
||||
rateLimitEnabled, targetMB,
|
||||
writeKp, writeKi, writeKd,
|
||||
readKp, readKi, readKd,
|
||||
maxWriteMs, maxReadMs,
|
||||
writeTarget, readTarget := cfg.GetRateLimitConfigValues()
|
||||
|
||||
if rateLimitEnabled {
|
||||
rlConfig := ratelimit.NewConfigFromValues(
|
||||
rateLimitEnabled, targetMB,
|
||||
writeKp, writeKi, writeKd,
|
||||
readKp, readKi, readKd,
|
||||
maxWriteMs, maxReadMs,
|
||||
writeTarget, readTarget,
|
||||
)
|
||||
|
||||
// Create appropriate monitor based on database type
|
||||
if badgerDB, ok := db.(*database.D); ok {
|
||||
limiter = ratelimit.NewBadgerLimiter(rlConfig, badgerDB.DB)
|
||||
log.I.F("rate limiter configured for Badger backend (target: %dMB)", targetMB)
|
||||
} else {
|
||||
// For Neo4j or other backends, create a disabled limiter for now
|
||||
// Neo4j monitor requires access to the querySem which is internal
|
||||
limiter = ratelimit.NewDisabledLimiter()
|
||||
log.I.F("rate limiter disabled for non-Badger backend")
|
||||
}
|
||||
} else {
|
||||
limiter = ratelimit.NewDisabledLimiter()
|
||||
}
|
||||
|
||||
// Start HTTP pprof server if enabled
|
||||
if cfg.PprofHTTP {
|
||||
pprofAddr := fmt.Sprintf("%s:%d", cfg.Listen, 6060)
|
||||
@@ -413,7 +445,7 @@ func main() {
|
||||
}()
|
||||
}
|
||||
|
||||
quit := app.Run(ctx, cfg, db)
|
||||
quit := app.Run(ctx, cfg, db, limiter)
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
|
||||
for {
|
||||
|
||||
257
pkg/database/IMPORT_MEMORY_OPTIMIZATION_PLAN.md
Normal file
257
pkg/database/IMPORT_MEMORY_OPTIMIZATION_PLAN.md
Normal file
@@ -0,0 +1,257 @@
|
||||
# Import Memory Optimization Plan
|
||||
|
||||
## Goal
|
||||
|
||||
Constrain import memory utilization to ≤1.5GB to ensure system disk cache flushing completes adequately before continuing.
|
||||
|
||||
## Test Results (Baseline)
|
||||
|
||||
- **File**: `wot_reference.jsonl` (2.7 GB, ~2.16 million events)
|
||||
- **System**: 15 GB RAM, Linux
|
||||
- **Events Saved**: 2,130,545
|
||||
- **Total Time**: 48 minutes 16 seconds
|
||||
- **Average Rate**: 736 events/sec
|
||||
- **Peak Memory**: ~6.4 GB (42% of system RAM)
|
||||
|
||||
### Memory Timeline (Baseline)
|
||||
|
||||
| Time | Memory (RSS) | Events | Notes |
|
||||
|------|--------------|--------|-------|
|
||||
| Start | 95 MB | 0 | Initial state |
|
||||
| +10 min | 2.7 GB | 283k | Warming up |
|
||||
| +20 min | 4.1 GB | 475k | Memory growing |
|
||||
| +30 min | 5.2 GB | 720k | Peak approaching |
|
||||
| +35 min | 5.9 GB | 485k | Near peak |
|
||||
| +40 min | 5.6 GB | 1.3M | GC recovered memory |
|
||||
| +48 min | 6.4 GB | 2.1M | Final (42% of RAM) |
|
||||
|
||||
## Root Causes of Memory Growth
|
||||
|
||||
### 1. Badger Internal Caches (configured in `database.go`)
|
||||
|
||||
- Block cache: 1024 MB default
|
||||
- Index cache: 512 MB default
|
||||
- Memtables: 8 × 16 MB = 128 MB
|
||||
- Total baseline: ~1.6 GB just for configured caches
|
||||
|
||||
### 2. Badger Write Buffers
|
||||
|
||||
- L0 tables buffer (8 tables × 16 MB)
|
||||
- Value log writes accumulate until compaction
|
||||
|
||||
### 3. No Backpressure in Import Loop
|
||||
|
||||
- Events are written continuously without waiting for compaction
|
||||
- `debug.FreeOSMemory()` only runs every 5 seconds
|
||||
- Badger buffers writes faster than disk can flush
|
||||
|
||||
### 4. Transaction Overhead
|
||||
|
||||
- Each `SaveEvent` creates a transaction
|
||||
- Transactions have overhead that accumulates
|
||||
|
||||
## Proposed Mitigations
|
||||
|
||||
### Phase 1: Reduce Badger Cache Configuration for Import
|
||||
|
||||
Add import-specific configuration options in `app/config/config.go`:
|
||||
|
||||
```go
|
||||
ImportBlockCacheMB int `env:"ORLY_IMPORT_BLOCK_CACHE_MB" default:"256"`
|
||||
ImportIndexCacheMB int `env:"ORLY_IMPORT_INDEX_CACHE_MB" default:"128"`
|
||||
ImportMemTableSize int `env:"ORLY_IMPORT_MEMTABLE_SIZE_MB" default:"8"`
|
||||
```
|
||||
|
||||
For a 1.5GB target:
|
||||
|
||||
| Component | Size | Notes |
|
||||
|-----------|------|-------|
|
||||
| Block cache | 256 MB | Reduced from 1024 MB |
|
||||
| Index cache | 128 MB | Reduced from 512 MB |
|
||||
| Memtables | 4 × 8 MB = 32 MB | Reduced from 8 × 16 MB |
|
||||
| Serial cache | ~20 MB | Unchanged |
|
||||
| Working memory | ~200 MB | Buffer for processing |
|
||||
| **Total** | **~636 MB** | Leaves headroom for 1.5GB target |
|
||||
|
||||
### Phase 2: Add Batching with Sync to Import Loop
|
||||
|
||||
Modify `import_utils.go` to batch writes and force sync:
|
||||
|
||||
```go
|
||||
const (
|
||||
importBatchSize = 500 // Events per batch
|
||||
importSyncInterval = 2000 // Events before forcing sync
|
||||
importMemCheckEvents = 1000 // Events between memory checks
|
||||
importMaxMemoryMB = 1400 // Target max memory (MB)
|
||||
)
|
||||
|
||||
// In processJSONLEventsWithPolicy:
|
||||
var batchCount int
|
||||
for scan.Scan() {
|
||||
// ... existing event processing ...
|
||||
|
||||
batchCount++
|
||||
count++
|
||||
|
||||
// Force sync periodically to flush writes to disk
|
||||
if batchCount >= importSyncInterval {
|
||||
d.DB.Sync() // Force write to disk
|
||||
batchCount = 0
|
||||
}
|
||||
|
||||
// Memory pressure check
|
||||
if count % importMemCheckEvents == 0 {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
heapMB := m.HeapAlloc / 1024 / 1024
|
||||
|
||||
if heapMB > importMaxMemoryMB {
|
||||
// Apply backpressure
|
||||
d.DB.Sync()
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
|
||||
// Wait for compaction to catch up
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 3: Use Batch Transactions
|
||||
|
||||
Instead of one transaction per event, batch multiple events:
|
||||
|
||||
```go
|
||||
// Accumulate events for batch write
|
||||
const txnBatchSize = 100
|
||||
|
||||
type pendingWrite struct {
|
||||
idxs [][]byte
|
||||
compactKey []byte
|
||||
compactVal []byte
|
||||
graphKeys [][]byte
|
||||
}
|
||||
|
||||
var pendingWrites []pendingWrite
|
||||
|
||||
// In the event processing loop
|
||||
pendingWrites = append(pendingWrites, pw)
|
||||
|
||||
if len(pendingWrites) >= txnBatchSize {
|
||||
err = d.Update(func(txn *badger.Txn) error {
|
||||
for _, pw := range pendingWrites {
|
||||
for _, key := range pw.idxs {
|
||||
txn.Set(key, nil)
|
||||
}
|
||||
txn.Set(pw.compactKey, pw.compactVal)
|
||||
for _, gk := range pw.graphKeys {
|
||||
txn.Set(gk, nil)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
pendingWrites = pendingWrites[:0]
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 4: Implement Adaptive Rate Limiting
|
||||
|
||||
```go
|
||||
type importRateLimiter struct {
|
||||
targetMemMB uint64
|
||||
checkInterval int
|
||||
baseDelay time.Duration
|
||||
maxDelay time.Duration
|
||||
}
|
||||
|
||||
func (r *importRateLimiter) maybeThrottle(eventCount int) {
|
||||
if eventCount % r.checkInterval != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
heapMB := m.HeapAlloc / 1024 / 1024
|
||||
|
||||
if heapMB > r.targetMemMB {
|
||||
// Calculate delay proportional to overage
|
||||
overage := float64(heapMB - r.targetMemMB) / float64(r.targetMemMB)
|
||||
delay := time.Duration(float64(r.baseDelay) * (1 + overage*10))
|
||||
if delay > r.maxDelay {
|
||||
delay = r.maxDelay
|
||||
}
|
||||
|
||||
// Force GC and wait
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. **Quick Win**: Add `d.DB.Sync()` call every N events in import loop
|
||||
2. **Configuration**: Add environment variables for import-specific cache sizes
|
||||
3. **Batching**: Implement batch transactions to reduce overhead
|
||||
4. **Adaptive**: Add memory-aware rate limiting
|
||||
|
||||
## Expected Results
|
||||
|
||||
| Approach | Memory Target | Throughput Impact |
|
||||
|----------|---------------|-------------------|
|
||||
| Current | ~6 GB peak | 736 events/sec |
|
||||
| Phase 1 (cache reduction) | ~2 GB | ~700 events/sec |
|
||||
| Phase 2 (sync + GC) | ~1.5 GB | ~500 events/sec |
|
||||
| Phase 3 (batching) | ~1.5 GB | ~600 events/sec |
|
||||
| Phase 4 (adaptive) | ~1.4 GB | Variable |
|
||||
|
||||
## Files to Modify
|
||||
|
||||
1. `app/config/config.go` - Add import-specific config options
|
||||
2. `pkg/database/database.go` - Add import mode with reduced caches
|
||||
3. `pkg/database/import_utils.go` - Add batching, sync, and rate limiting
|
||||
4. `pkg/database/save-event.go` - Add batch save method (optional, for Phase 3)
|
||||
|
||||
## Environment Variables (Proposed)
|
||||
|
||||
```bash
|
||||
# Import-specific cache settings (only apply during import operations)
|
||||
ORLY_IMPORT_BLOCK_CACHE_MB=256 # Block cache size during import
|
||||
ORLY_IMPORT_INDEX_CACHE_MB=128 # Index cache size during import
|
||||
ORLY_IMPORT_MEMTABLE_SIZE_MB=8 # Memtable size during import
|
||||
|
||||
# Import rate limiting
|
||||
ORLY_IMPORT_SYNC_INTERVAL=2000 # Events between forced syncs
|
||||
ORLY_IMPORT_MAX_MEMORY_MB=1400 # Target max memory during import
|
||||
ORLY_IMPORT_BATCH_SIZE=100 # Events per transaction batch
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- The adaptive rate limiting (Phase 4) is the most robust solution but adds complexity
|
||||
- Phase 2 alone should achieve the 1.5GB target with acceptable throughput
|
||||
- Batch transactions (Phase 3) can improve throughput but require refactoring `SaveEvent`
|
||||
- Consider making these settings configurable so users can tune for their hardware
|
||||
|
||||
## Test Command
|
||||
|
||||
To re-run the import test with memory monitoring:
|
||||
|
||||
```bash
|
||||
# Start relay with import-optimized settings
|
||||
export ORLY_DATA_DIR=/tmp/orly-import-test
|
||||
export ORLY_ACL_MODE=none
|
||||
export ORLY_PORT=10548
|
||||
export ORLY_LOG_LEVEL=info
|
||||
./orly &
|
||||
|
||||
# Upload test file
|
||||
curl -X POST \
|
||||
-F "file=@/path/to/wot_reference.jsonl" \
|
||||
http://localhost:10548/api/import
|
||||
|
||||
# Monitor memory
|
||||
watch -n 5 'ps -p $(pgrep orly) -o pid,rss,pmem --no-headers'
|
||||
```
|
||||
@@ -6,9 +6,11 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"lol.mleku.dev/chk"
|
||||
"lol.mleku.dev/log"
|
||||
"next.orly.dev/pkg/database/indexes"
|
||||
"next.orly.dev/pkg/database/indexes/types"
|
||||
"git.mleku.dev/mleku/nostr/encoders/event"
|
||||
@@ -22,6 +24,14 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
evB := make([]byte, 0, units.Mb)
|
||||
evBuf := bytes.NewBuffer(evB)
|
||||
|
||||
// Performance tracking
|
||||
startTime := time.Now()
|
||||
var eventCount, bytesWritten int64
|
||||
lastLogTime := startTime
|
||||
const logInterval = 5 * time.Second
|
||||
|
||||
log.I.F("export: starting export operation")
|
||||
|
||||
// Create resolver for compact event decoding
|
||||
resolver := NewDatabaseSerialResolver(d, d.serialCache)
|
||||
|
||||
@@ -86,7 +96,8 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
}
|
||||
|
||||
// Serialize the event to JSON and write it to the output
|
||||
if _, err = w.Write(ev.Serialize()); chk.E(err) {
|
||||
data := ev.Serialize()
|
||||
if _, err = w.Write(data); chk.E(err) {
|
||||
ev.Free()
|
||||
return
|
||||
}
|
||||
@@ -94,7 +105,19 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
ev.Free()
|
||||
return
|
||||
}
|
||||
bytesWritten += int64(len(data) + 1)
|
||||
eventCount++
|
||||
ev.Free()
|
||||
|
||||
// Progress logging every logInterval
|
||||
if time.Since(lastLogTime) >= logInterval {
|
||||
elapsed := time.Since(startTime)
|
||||
eventsPerSec := float64(eventCount) / elapsed.Seconds()
|
||||
mbPerSec := float64(bytesWritten) / elapsed.Seconds() / 1024 / 1024
|
||||
log.I.F("export: progress %d events, %.2f MB written, %.0f events/sec, %.2f MB/sec",
|
||||
eventCount, float64(bytesWritten)/1024/1024, eventsPerSec, mbPerSec)
|
||||
lastLogTime = time.Now()
|
||||
}
|
||||
}
|
||||
it.Close()
|
||||
|
||||
@@ -133,7 +156,8 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
}
|
||||
|
||||
// Serialize the event to JSON and write it to the output
|
||||
if _, err = w.Write(ev.Serialize()); chk.E(err) {
|
||||
data := ev.Serialize()
|
||||
if _, err = w.Write(data); chk.E(err) {
|
||||
ev.Free()
|
||||
return
|
||||
}
|
||||
@@ -141,7 +165,19 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
ev.Free()
|
||||
return
|
||||
}
|
||||
bytesWritten += int64(len(data) + 1)
|
||||
eventCount++
|
||||
ev.Free()
|
||||
|
||||
// Progress logging every logInterval
|
||||
if time.Since(lastLogTime) >= logInterval {
|
||||
elapsed := time.Since(startTime)
|
||||
eventsPerSec := float64(eventCount) / elapsed.Seconds()
|
||||
mbPerSec := float64(bytesWritten) / elapsed.Seconds() / 1024 / 1024
|
||||
log.I.F("export: progress %d events, %.2f MB written, %.0f events/sec, %.2f MB/sec",
|
||||
eventCount, float64(bytesWritten)/1024/1024, eventsPerSec, mbPerSec)
|
||||
lastLogTime = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
@@ -149,8 +185,16 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Final export summary
|
||||
elapsed := time.Since(startTime)
|
||||
eventsPerSec := float64(eventCount) / elapsed.Seconds()
|
||||
mbPerSec := float64(bytesWritten) / elapsed.Seconds() / 1024 / 1024
|
||||
log.I.F("export: completed - %d events, %.2f MB in %v (%.0f events/sec, %.2f MB/sec)",
|
||||
eventCount, float64(bytesWritten)/1024/1024, elapsed.Round(time.Millisecond), eventsPerSec, mbPerSec)
|
||||
} else {
|
||||
// Export events for specific pubkeys
|
||||
log.I.F("export: exporting events for %d pubkeys", len(pubkeys))
|
||||
for _, pubkey := range pubkeys {
|
||||
if err = d.View(
|
||||
func(txn *badger.Txn) (err error) {
|
||||
@@ -187,7 +231,8 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
}
|
||||
|
||||
// Serialize the event to JSON and write it to the output
|
||||
if _, err = w.Write(ev.Serialize()); chk.E(err) {
|
||||
data := ev.Serialize()
|
||||
if _, err = w.Write(data); chk.E(err) {
|
||||
ev.Free()
|
||||
continue
|
||||
}
|
||||
@@ -195,7 +240,19 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
ev.Free()
|
||||
continue
|
||||
}
|
||||
bytesWritten += int64(len(data) + 1)
|
||||
eventCount++
|
||||
ev.Free()
|
||||
|
||||
// Progress logging every logInterval
|
||||
if time.Since(lastLogTime) >= logInterval {
|
||||
elapsed := time.Since(startTime)
|
||||
eventsPerSec := float64(eventCount) / elapsed.Seconds()
|
||||
mbPerSec := float64(bytesWritten) / elapsed.Seconds() / 1024 / 1024
|
||||
log.I.F("export: progress %d events, %.2f MB written, %.0f events/sec, %.2f MB/sec",
|
||||
eventCount, float64(bytesWritten)/1024/1024, eventsPerSec, mbPerSec)
|
||||
lastLogTime = time.Now()
|
||||
}
|
||||
}
|
||||
return
|
||||
},
|
||||
@@ -203,5 +260,12 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Final export summary for pubkey export
|
||||
elapsed := time.Since(startTime)
|
||||
eventsPerSec := float64(eventCount) / elapsed.Seconds()
|
||||
mbPerSec := float64(bytesWritten) / elapsed.Seconds() / 1024 / 1024
|
||||
log.I.F("export: completed - %d events, %.2f MB in %v (%.0f events/sec, %.2f MB/sec)",
|
||||
eventCount, float64(bytesWritten)/1024/1024, elapsed.Round(time.Millisecond), eventsPerSec, mbPerSec)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lol.mleku.dev/chk"
|
||||
"lol.mleku.dev/log"
|
||||
@@ -20,6 +21,9 @@ const maxLen = 500000000
|
||||
|
||||
// ImportEventsFromReader imports events from an io.Reader containing JSONL data
|
||||
func (d *D) ImportEventsFromReader(ctx context.Context, rr io.Reader) error {
|
||||
startTime := time.Now()
|
||||
log.I.F("import: starting import operation")
|
||||
|
||||
// store to disk so we can return fast
|
||||
tmpPath := os.TempDir() + string(os.PathSeparator) + "orly"
|
||||
os.MkdirAll(tmpPath, 0700)
|
||||
@@ -29,15 +33,27 @@ func (d *D) ImportEventsFromReader(ctx context.Context, rr io.Reader) error {
|
||||
}
|
||||
defer os.Remove(tmp.Name()) // Clean up temp file when done
|
||||
|
||||
log.I.F("buffering upload to %s", tmp.Name())
|
||||
if _, err = io.Copy(tmp, rr); chk.E(err) {
|
||||
log.I.F("import: buffering upload to %s", tmp.Name())
|
||||
bufferStart := time.Now()
|
||||
bytesBuffered, err := io.Copy(tmp, rr)
|
||||
if chk.E(err) {
|
||||
return err
|
||||
}
|
||||
bufferElapsed := time.Since(bufferStart)
|
||||
log.I.F("import: buffered %.2f MB in %v (%.2f MB/sec)",
|
||||
float64(bytesBuffered)/1024/1024, bufferElapsed.Round(time.Millisecond),
|
||||
float64(bytesBuffered)/bufferElapsed.Seconds()/1024/1024)
|
||||
|
||||
if _, err = tmp.Seek(0, 0); chk.E(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return d.processJSONLEvents(ctx, tmp)
|
||||
processErr := d.processJSONLEvents(ctx, tmp)
|
||||
|
||||
totalElapsed := time.Since(startTime)
|
||||
log.I.F("import: total operation time: %v", totalElapsed.Round(time.Millisecond))
|
||||
|
||||
return processErr
|
||||
}
|
||||
|
||||
// ImportEventsFromStrings imports events from a slice of JSON strings with policy filtering
|
||||
@@ -59,11 +75,16 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
|
||||
scanBuf := make([]byte, maxLen)
|
||||
scan.Buffer(scanBuf, maxLen)
|
||||
|
||||
var count, total int
|
||||
// Performance tracking
|
||||
startTime := time.Now()
|
||||
lastLogTime := startTime
|
||||
const logInterval = 5 * time.Second
|
||||
|
||||
var count, total, skipped, policyRejected, unmarshalErrors, saveErrors int
|
||||
for scan.Scan() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.I.F("context closed")
|
||||
log.I.F("import: context closed after %d events", count)
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
@@ -71,6 +92,7 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
|
||||
b := scan.Bytes()
|
||||
total += len(b) + 1
|
||||
if len(b) < 1 {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -78,6 +100,7 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
|
||||
if _, err := ev.Unmarshal(b); err != nil {
|
||||
// return the pooled buffer on error
|
||||
ev.Free()
|
||||
unmarshalErrors++
|
||||
log.W.F("failed to unmarshal event: %v", err)
|
||||
continue
|
||||
}
|
||||
@@ -90,11 +113,13 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
|
||||
if policyErr != nil {
|
||||
log.W.F("policy check failed for event %x: %v", ev.ID, policyErr)
|
||||
ev.Free()
|
||||
policyRejected++
|
||||
continue
|
||||
}
|
||||
if !allowed {
|
||||
log.D.F("policy rejected event %x during sync import", ev.ID)
|
||||
ev.Free()
|
||||
policyRejected++
|
||||
continue
|
||||
}
|
||||
log.D.F("policy allowed event %x during sync import", ev.ID)
|
||||
@@ -103,6 +128,7 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
|
||||
if _, err := d.SaveEvent(ctx, ev); err != nil {
|
||||
// return the pooled buffer on error paths too
|
||||
ev.Free()
|
||||
saveErrors++
|
||||
log.W.F("failed to save event: %v", err)
|
||||
continue
|
||||
}
|
||||
@@ -111,13 +137,30 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
|
||||
ev.Free()
|
||||
b = nil
|
||||
count++
|
||||
if count%100 == 0 {
|
||||
log.I.F("processed %d events", count)
|
||||
|
||||
// Progress logging every logInterval
|
||||
if time.Since(lastLogTime) >= logInterval {
|
||||
elapsed := time.Since(startTime)
|
||||
eventsPerSec := float64(count) / elapsed.Seconds()
|
||||
mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024
|
||||
log.I.F("import: progress %d events saved, %.2f MB read, %.0f events/sec, %.2f MB/sec",
|
||||
count, float64(total)/1024/1024, eventsPerSec, mbPerSec)
|
||||
lastLogTime = time.Now()
|
||||
debug.FreeOSMemory()
|
||||
}
|
||||
}
|
||||
|
||||
log.I.F("read %d bytes and saved %d events", total, count)
|
||||
// Final summary
|
||||
elapsed := time.Since(startTime)
|
||||
eventsPerSec := float64(count) / elapsed.Seconds()
|
||||
mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024
|
||||
log.I.F("import: completed - %d events saved, %.2f MB in %v (%.0f events/sec, %.2f MB/sec)",
|
||||
count, float64(total)/1024/1024, elapsed.Round(time.Millisecond), eventsPerSec, mbPerSec)
|
||||
if unmarshalErrors > 0 || saveErrors > 0 || policyRejected > 0 || skipped > 0 {
|
||||
log.I.F("import: stats - %d unmarshal errors, %d save errors, %d policy rejected, %d skipped empty lines",
|
||||
unmarshalErrors, saveErrors, policyRejected, skipped)
|
||||
}
|
||||
|
||||
if err := scan.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
58
pkg/interfaces/loadmonitor/loadmonitor.go
Normal file
58
pkg/interfaces/loadmonitor/loadmonitor.go
Normal file
@@ -0,0 +1,58 @@
|
||||
// Package loadmonitor defines the interface for database load monitoring.
|
||||
// This allows different database backends to provide their own load metrics
|
||||
// while the rate limiter remains database-agnostic.
|
||||
package loadmonitor
|
||||
|
||||
import "time"
|
||||
|
||||
// Metrics contains load metrics from a database backend.
|
||||
// All values are normalized to 0.0-1.0 where 0 means no load and 1 means at capacity.
|
||||
type Metrics struct {
|
||||
// MemoryPressure indicates memory usage relative to a target limit (0.0-1.0+).
|
||||
// Values above 1.0 indicate the target has been exceeded.
|
||||
MemoryPressure float64
|
||||
|
||||
// WriteLoad indicates the write-side load level (0.0-1.0).
|
||||
// For Badger: L0 tables and compaction score
|
||||
// For Neo4j: active write transactions
|
||||
WriteLoad float64
|
||||
|
||||
// ReadLoad indicates the read-side load level (0.0-1.0).
|
||||
// For Badger: cache hit ratio (inverted)
|
||||
// For Neo4j: active read transactions
|
||||
ReadLoad float64
|
||||
|
||||
// QueryLatency is the recent average query latency.
|
||||
QueryLatency time.Duration
|
||||
|
||||
// WriteLatency is the recent average write latency.
|
||||
WriteLatency time.Duration
|
||||
|
||||
// Timestamp is when these metrics were collected.
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// Monitor defines the interface for database load monitoring.
|
||||
// Implementations are database-specific (Badger, Neo4j, etc.).
|
||||
type Monitor interface {
|
||||
// GetMetrics returns the current load metrics.
|
||||
// This should be efficient as it may be called frequently.
|
||||
GetMetrics() Metrics
|
||||
|
||||
// RecordQueryLatency records a query latency sample for averaging.
|
||||
RecordQueryLatency(latency time.Duration)
|
||||
|
||||
// RecordWriteLatency records a write latency sample for averaging.
|
||||
RecordWriteLatency(latency time.Duration)
|
||||
|
||||
// SetMemoryTarget sets the target memory limit in bytes.
|
||||
// Memory pressure is calculated relative to this target.
|
||||
SetMemoryTarget(bytes uint64)
|
||||
|
||||
// Start begins background metric collection.
|
||||
// Returns a channel that will be closed when the monitor is stopped.
|
||||
Start() <-chan struct{}
|
||||
|
||||
// Stop halts background metric collection.
|
||||
Stop()
|
||||
}
|
||||
133
pkg/interfaces/pid/pid.go
Normal file
133
pkg/interfaces/pid/pid.go
Normal file
@@ -0,0 +1,133 @@
|
||||
// Package pid defines interfaces for PID controller process variable sources.
|
||||
// This abstraction allows the PID controller to be used for any dynamic
|
||||
// adjustment scenario - rate limiting, PoW difficulty adjustment, etc.
|
||||
package pid
|
||||
|
||||
import "time"
|
||||
|
||||
// ProcessVariable represents a measurable quantity that the PID controller
|
||||
// regulates. Implementations provide the current value and optional metadata.
|
||||
type ProcessVariable interface {
|
||||
// Value returns the current process variable value.
|
||||
// The value should typically be normalized to a range where the setpoint
|
||||
// makes sense (e.g., 0.0-1.0 for percentage-based control, or absolute
|
||||
// values for things like hash rate or block time).
|
||||
Value() float64
|
||||
|
||||
// Timestamp returns when this measurement was taken.
|
||||
// This is used for derivative calculations and staleness detection.
|
||||
Timestamp() time.Time
|
||||
}
|
||||
|
||||
// Source provides process variable measurements to the PID controller.
|
||||
// Implementations are domain-specific (e.g., memory monitor, hash rate tracker).
|
||||
type Source interface {
|
||||
// Sample returns the current process variable measurement.
|
||||
// This should be efficient as it may be called frequently.
|
||||
Sample() ProcessVariable
|
||||
|
||||
// Name returns a human-readable name for this source (for logging/debugging).
|
||||
Name() string
|
||||
}
|
||||
|
||||
// Output represents the result of a PID controller update.
|
||||
type Output interface {
|
||||
// Value returns the computed output value.
|
||||
// The interpretation depends on the application:
|
||||
// - For rate limiting: delay in seconds
|
||||
// - For PoW difficulty: difficulty adjustment factor
|
||||
// - For temperature control: heater power level
|
||||
Value() float64
|
||||
|
||||
// Clamped returns true if the output was clamped to limits.
|
||||
Clamped() bool
|
||||
|
||||
// Components returns the individual P, I, D contributions for debugging.
|
||||
Components() (p, i, d float64)
|
||||
}
|
||||
|
||||
// Controller defines the interface for a PID controller.
|
||||
// This allows for different controller implementations (standard PID,
|
||||
// PID with filtered derivative, adaptive PID, etc.).
|
||||
type Controller interface {
|
||||
// Update computes the controller output based on the current process variable.
|
||||
// Returns the computed output.
|
||||
Update(pv ProcessVariable) Output
|
||||
|
||||
// UpdateValue is a convenience method that takes a raw float64 value.
|
||||
// Uses the current time as the timestamp.
|
||||
UpdateValue(value float64) Output
|
||||
|
||||
// Reset clears all internal state (integral accumulator, previous values).
|
||||
Reset()
|
||||
|
||||
// SetSetpoint updates the target value.
|
||||
SetSetpoint(setpoint float64)
|
||||
|
||||
// Setpoint returns the current setpoint.
|
||||
Setpoint() float64
|
||||
|
||||
// SetGains updates the PID gains.
|
||||
SetGains(kp, ki, kd float64)
|
||||
|
||||
// Gains returns the current PID gains.
|
||||
Gains() (kp, ki, kd float64)
|
||||
}
|
||||
|
||||
// Tuning holds PID tuning parameters.
|
||||
// This can be used for configuration or auto-tuning.
|
||||
type Tuning struct {
|
||||
Kp float64 // Proportional gain
|
||||
Ki float64 // Integral gain
|
||||
Kd float64 // Derivative gain
|
||||
|
||||
Setpoint float64 // Target value
|
||||
|
||||
// Derivative filtering (0.0-1.0, lower = more filtering)
|
||||
DerivativeFilterAlpha float64
|
||||
|
||||
// Anti-windup limits for integral term
|
||||
IntegralMin float64
|
||||
IntegralMax float64
|
||||
|
||||
// Output limits
|
||||
OutputMin float64
|
||||
OutputMax float64
|
||||
}
|
||||
|
||||
// DefaultTuning returns sensible defaults for a normalized (0-1) process variable.
|
||||
func DefaultTuning() Tuning {
|
||||
return Tuning{
|
||||
Kp: 0.5,
|
||||
Ki: 0.1,
|
||||
Kd: 0.05,
|
||||
Setpoint: 0.5,
|
||||
DerivativeFilterAlpha: 0.2,
|
||||
IntegralMin: -10.0,
|
||||
IntegralMax: 10.0,
|
||||
OutputMin: 0.0,
|
||||
OutputMax: 1.0,
|
||||
}
|
||||
}
|
||||
|
||||
// SimpleProcessVariable is a basic implementation of ProcessVariable.
|
||||
type SimpleProcessVariable struct {
|
||||
V float64
|
||||
T time.Time
|
||||
}
|
||||
|
||||
// Value returns the process variable value.
|
||||
func (p SimpleProcessVariable) Value() float64 { return p.V }
|
||||
|
||||
// Timestamp returns when this measurement was taken.
|
||||
func (p SimpleProcessVariable) Timestamp() time.Time { return p.T }
|
||||
|
||||
// NewProcessVariable creates a SimpleProcessVariable with the current time.
|
||||
func NewProcessVariable(value float64) SimpleProcessVariable {
|
||||
return SimpleProcessVariable{V: value, T: time.Now()}
|
||||
}
|
||||
|
||||
// NewProcessVariableAt creates a SimpleProcessVariable with a specific time.
|
||||
func NewProcessVariableAt(value float64, t time.Time) SimpleProcessVariable {
|
||||
return SimpleProcessVariable{V: value, T: t}
|
||||
}
|
||||
266
pkg/pid/controller.go
Normal file
266
pkg/pid/controller.go
Normal file
@@ -0,0 +1,266 @@
|
||||
// Package pid provides a generic PID controller implementation with filtered derivative.
|
||||
//
|
||||
// This package implements a Proportional-Integral-Derivative controller suitable
|
||||
// for various dynamic adjustment scenarios:
|
||||
// - Rate limiting (memory/load-based throttling)
|
||||
// - PoW difficulty adjustment (block time targeting)
|
||||
// - Temperature control
|
||||
// - Motor speed control
|
||||
// - Any system requiring feedback-based regulation
|
||||
//
|
||||
// The controller features:
|
||||
// - Low-pass filtered derivative to suppress high-frequency noise
|
||||
// - Anti-windup on the integral term to prevent saturation
|
||||
// - Configurable output clamping
|
||||
// - Thread-safe operation
|
||||
//
|
||||
// # Control Theory Background
|
||||
//
|
||||
// The PID controller computes an output based on the error between the current
|
||||
// process variable and a target setpoint:
|
||||
//
|
||||
// output = Kp*error + Ki*∫error*dt + Kd*d(filtered_error)/dt
|
||||
//
|
||||
// Where:
|
||||
// - Proportional (P): Immediate response proportional to current error
|
||||
// - Integral (I): Accumulated error to eliminate steady-state offset
|
||||
// - Derivative (D): Rate of change to anticipate future error (filtered)
|
||||
//
|
||||
// # Filtered Derivative
|
||||
//
|
||||
// Raw derivative amplifies high-frequency noise. This implementation applies
|
||||
// an exponential moving average (low-pass filter) before computing the derivative:
|
||||
//
|
||||
// filtered_error = α*current_error + (1-α)*previous_filtered_error
|
||||
// derivative = (filtered_error - previous_filtered_error) / dt
|
||||
//
|
||||
// Lower α values provide stronger filtering (recommended: 0.1-0.3).
|
||||
package pid
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
pidif "next.orly.dev/pkg/interfaces/pid"
|
||||
)
|
||||
|
||||
// Controller implements a PID controller with filtered derivative.
|
||||
// It is safe for concurrent use.
|
||||
type Controller struct {
|
||||
// Configuration (protected by mutex for dynamic updates)
|
||||
mu sync.Mutex
|
||||
tuning pidif.Tuning
|
||||
|
||||
// Internal state
|
||||
integral float64
|
||||
prevError float64
|
||||
prevFilteredError float64
|
||||
lastUpdate time.Time
|
||||
initialized bool
|
||||
}
|
||||
|
||||
// Compile-time check that Controller implements pidif.Controller
|
||||
var _ pidif.Controller = (*Controller)(nil)
|
||||
|
||||
// output implements pidif.Output
|
||||
type output struct {
|
||||
value float64
|
||||
clamped bool
|
||||
pTerm float64
|
||||
iTerm float64
|
||||
dTerm float64
|
||||
}
|
||||
|
||||
func (o output) Value() float64 { return o.value }
|
||||
func (o output) Clamped() bool { return o.clamped }
|
||||
func (o output) Components() (p, i, d float64) { return o.pTerm, o.iTerm, o.dTerm }
|
||||
|
||||
// New creates a new PID controller with the given tuning parameters.
|
||||
func New(tuning pidif.Tuning) *Controller {
|
||||
return &Controller{tuning: tuning}
|
||||
}
|
||||
|
||||
// NewWithGains creates a new PID controller with specified gains and defaults for other parameters.
|
||||
func NewWithGains(kp, ki, kd, setpoint float64) *Controller {
|
||||
tuning := pidif.DefaultTuning()
|
||||
tuning.Kp = kp
|
||||
tuning.Ki = ki
|
||||
tuning.Kd = kd
|
||||
tuning.Setpoint = setpoint
|
||||
return &Controller{tuning: tuning}
|
||||
}
|
||||
|
||||
// NewDefault creates a new PID controller with default tuning.
|
||||
func NewDefault() *Controller {
|
||||
return &Controller{tuning: pidif.DefaultTuning()}
|
||||
}
|
||||
|
||||
// Update computes the controller output based on the current process variable.
|
||||
func (c *Controller) Update(pv pidif.ProcessVariable) pidif.Output {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
now := pv.Timestamp()
|
||||
value := pv.Value()
|
||||
|
||||
// Initialize on first call
|
||||
if !c.initialized {
|
||||
c.lastUpdate = now
|
||||
c.prevError = value - c.tuning.Setpoint
|
||||
c.prevFilteredError = c.prevError
|
||||
c.initialized = true
|
||||
return output{value: 0, clamped: false}
|
||||
}
|
||||
|
||||
// Calculate time delta
|
||||
dt := now.Sub(c.lastUpdate).Seconds()
|
||||
if dt <= 0 {
|
||||
dt = 0.001 // Minimum 1ms to avoid division by zero
|
||||
}
|
||||
c.lastUpdate = now
|
||||
|
||||
// Calculate current error (positive when above setpoint)
|
||||
err := value - c.tuning.Setpoint
|
||||
|
||||
// Proportional term
|
||||
pTerm := c.tuning.Kp * err
|
||||
|
||||
// Integral term with anti-windup
|
||||
c.integral += err * dt
|
||||
c.integral = clamp(c.integral, c.tuning.IntegralMin, c.tuning.IntegralMax)
|
||||
iTerm := c.tuning.Ki * c.integral
|
||||
|
||||
// Derivative term with low-pass filter
|
||||
alpha := c.tuning.DerivativeFilterAlpha
|
||||
if alpha <= 0 {
|
||||
alpha = 0.2 // Default if not set
|
||||
}
|
||||
filteredError := alpha*err + (1-alpha)*c.prevFilteredError
|
||||
|
||||
var dTerm float64
|
||||
if dt > 0 {
|
||||
dTerm = c.tuning.Kd * (filteredError - c.prevFilteredError) / dt
|
||||
}
|
||||
|
||||
// Update previous values
|
||||
c.prevError = err
|
||||
c.prevFilteredError = filteredError
|
||||
|
||||
// Compute total output
|
||||
rawOutput := pTerm + iTerm + dTerm
|
||||
clampedOutput := clamp(rawOutput, c.tuning.OutputMin, c.tuning.OutputMax)
|
||||
|
||||
return output{
|
||||
value: clampedOutput,
|
||||
clamped: rawOutput != clampedOutput,
|
||||
pTerm: pTerm,
|
||||
iTerm: iTerm,
|
||||
dTerm: dTerm,
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateValue is a convenience method that takes a raw float64 value.
|
||||
func (c *Controller) UpdateValue(value float64) pidif.Output {
|
||||
return c.Update(pidif.NewProcessVariable(value))
|
||||
}
|
||||
|
||||
// Reset clears all internal state.
|
||||
func (c *Controller) Reset() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.integral = 0
|
||||
c.prevError = 0
|
||||
c.prevFilteredError = 0
|
||||
c.initialized = false
|
||||
}
|
||||
|
||||
// SetSetpoint updates the target value.
|
||||
func (c *Controller) SetSetpoint(setpoint float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tuning.Setpoint = setpoint
|
||||
}
|
||||
|
||||
// Setpoint returns the current setpoint.
|
||||
func (c *Controller) Setpoint() float64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.tuning.Setpoint
|
||||
}
|
||||
|
||||
// SetGains updates the PID gains.
|
||||
func (c *Controller) SetGains(kp, ki, kd float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tuning.Kp = kp
|
||||
c.tuning.Ki = ki
|
||||
c.tuning.Kd = kd
|
||||
}
|
||||
|
||||
// Gains returns the current PID gains.
|
||||
func (c *Controller) Gains() (kp, ki, kd float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.tuning.Kp, c.tuning.Ki, c.tuning.Kd
|
||||
}
|
||||
|
||||
// SetOutputLimits updates the output clamping limits.
|
||||
func (c *Controller) SetOutputLimits(min, max float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tuning.OutputMin = min
|
||||
c.tuning.OutputMax = max
|
||||
}
|
||||
|
||||
// SetIntegralLimits updates the anti-windup limits.
|
||||
func (c *Controller) SetIntegralLimits(min, max float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tuning.IntegralMin = min
|
||||
c.tuning.IntegralMax = max
|
||||
}
|
||||
|
||||
// SetDerivativeFilter updates the derivative filter coefficient.
|
||||
// Lower values provide stronger filtering (0.1-0.3 recommended).
|
||||
func (c *Controller) SetDerivativeFilter(alpha float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tuning.DerivativeFilterAlpha = alpha
|
||||
}
|
||||
|
||||
// Tuning returns a copy of the current tuning parameters.
|
||||
func (c *Controller) Tuning() pidif.Tuning {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.tuning
|
||||
}
|
||||
|
||||
// SetTuning updates all tuning parameters at once.
|
||||
func (c *Controller) SetTuning(tuning pidif.Tuning) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tuning = tuning
|
||||
}
|
||||
|
||||
// State returns the current internal state for monitoring/debugging.
|
||||
func (c *Controller) State() (integral, prevError, prevFilteredError float64, initialized bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.integral, c.prevError, c.prevFilteredError, c.initialized
|
||||
}
|
||||
|
||||
// clamp restricts a value to the range [min, max].
|
||||
func clamp(value, min, max float64) float64 {
|
||||
if math.IsNaN(value) {
|
||||
return 0
|
||||
}
|
||||
if value < min {
|
||||
return min
|
||||
}
|
||||
if value > max {
|
||||
return max
|
||||
}
|
||||
return value
|
||||
}
|
||||
402
pkg/pid/controller_test.go
Normal file
402
pkg/pid/controller_test.go
Normal file
@@ -0,0 +1,402 @@
|
||||
package pid
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pidif "next.orly.dev/pkg/interfaces/pid"
|
||||
)
|
||||
|
||||
func TestController_BasicOperation(t *testing.T) {
|
||||
ctrl := New(RateLimitWriteTuning())
|
||||
|
||||
// First call should return 0 (initialization)
|
||||
out := ctrl.UpdateValue(0.5)
|
||||
if out.Value() != 0 {
|
||||
t.Errorf("expected 0 on first call, got %v", out.Value())
|
||||
}
|
||||
|
||||
// Sleep a bit to ensure dt > 0
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Process variable below setpoint (0.5 < 0.85) should return 0 or negative (clamped to 0)
|
||||
out = ctrl.UpdateValue(0.5)
|
||||
if out.Value() != 0 {
|
||||
t.Errorf("expected 0 when below setpoint, got %v", out.Value())
|
||||
}
|
||||
|
||||
// Process variable above setpoint should return positive output
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
out = ctrl.UpdateValue(0.95) // 0.95 > 0.85 setpoint
|
||||
if out.Value() <= 0 {
|
||||
t.Errorf("expected positive output when above setpoint, got %v", out.Value())
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_IntegralAccumulation(t *testing.T) {
|
||||
tuning := pidif.Tuning{
|
||||
Kp: 0.5,
|
||||
Ki: 0.5, // High Ki
|
||||
Kd: 0.0, // No Kd
|
||||
Setpoint: 0.5,
|
||||
DerivativeFilterAlpha: 0.2,
|
||||
IntegralMin: -10,
|
||||
IntegralMax: 10,
|
||||
OutputMin: 0,
|
||||
OutputMax: 1.0,
|
||||
}
|
||||
ctrl := New(tuning)
|
||||
|
||||
// Initialize
|
||||
ctrl.UpdateValue(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Continuously above setpoint should accumulate integral
|
||||
for i := 0; i < 10; i++ {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
ctrl.UpdateValue(0.8) // 0.3 above setpoint
|
||||
}
|
||||
|
||||
integral, _, _, _ := ctrl.State()
|
||||
if integral <= 0 {
|
||||
t.Errorf("expected positive integral after sustained error, got %v", integral)
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_FilteredDerivative(t *testing.T) {
|
||||
tuning := pidif.Tuning{
|
||||
Kp: 0.0,
|
||||
Ki: 0.0,
|
||||
Kd: 1.0, // Only Kd
|
||||
Setpoint: 0.5,
|
||||
DerivativeFilterAlpha: 0.5, // 50% filtering
|
||||
IntegralMin: -10,
|
||||
IntegralMax: 10,
|
||||
OutputMin: 0,
|
||||
OutputMax: 1.0,
|
||||
}
|
||||
ctrl := New(tuning)
|
||||
|
||||
// Initialize with low value
|
||||
ctrl.UpdateValue(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Second call with same value - derivative should be near zero
|
||||
ctrl.UpdateValue(0.5)
|
||||
_, _, prevFiltered, _ := ctrl.State()
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Big jump - filtered derivative should be dampened
|
||||
out := ctrl.UpdateValue(1.0)
|
||||
|
||||
// The filtered derivative should cause some response, but dampened
|
||||
if out.Value() < 0 {
|
||||
t.Errorf("expected non-negative output, got %v", out.Value())
|
||||
}
|
||||
|
||||
_, _, newFiltered, _ := ctrl.State()
|
||||
// Filtered error should have moved toward the new error but not fully
|
||||
if newFiltered <= prevFiltered {
|
||||
t.Errorf("filtered error should increase with rising process variable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_AntiWindup(t *testing.T) {
|
||||
tuning := pidif.Tuning{
|
||||
Kp: 0.0,
|
||||
Ki: 1.0, // Only Ki
|
||||
Kd: 0.0,
|
||||
Setpoint: 0.5,
|
||||
DerivativeFilterAlpha: 0.2,
|
||||
IntegralMin: -1.0, // Tight integral bounds
|
||||
IntegralMax: 1.0,
|
||||
OutputMin: 0,
|
||||
OutputMax: 10.0, // Wide output bounds
|
||||
}
|
||||
ctrl := New(tuning)
|
||||
|
||||
// Initialize
|
||||
ctrl.UpdateValue(0.5)
|
||||
|
||||
// Drive the integral to its limit
|
||||
for i := 0; i < 100; i++ {
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
ctrl.UpdateValue(1.0) // Large positive error
|
||||
}
|
||||
|
||||
integral, _, _, _ := ctrl.State()
|
||||
if integral > 1.0 {
|
||||
t.Errorf("integral should be clamped at 1.0, got %v", integral)
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_Reset(t *testing.T) {
|
||||
ctrl := New(RateLimitWriteTuning())
|
||||
|
||||
// Build up some state
|
||||
ctrl.UpdateValue(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
ctrl.UpdateValue(0.9)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
ctrl.UpdateValue(0.95)
|
||||
|
||||
// Reset
|
||||
ctrl.Reset()
|
||||
|
||||
integral, prevErr, prevFiltered, initialized := ctrl.State()
|
||||
if integral != 0 || prevErr != 0 || prevFiltered != 0 || initialized {
|
||||
t.Errorf("expected all state to be zero after reset, got integral=%v, prevErr=%v, prevFiltered=%v, initialized=%v",
|
||||
integral, prevErr, prevFiltered, initialized)
|
||||
}
|
||||
|
||||
// Next call should behave like first call
|
||||
out := ctrl.UpdateValue(0.9)
|
||||
if out.Value() != 0 {
|
||||
t.Errorf("expected 0 on first call after reset, got %v", out.Value())
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_SetGains(t *testing.T) {
|
||||
ctrl := New(RateLimitWriteTuning())
|
||||
|
||||
// Change gains
|
||||
ctrl.SetGains(1.0, 0.5, 0.1)
|
||||
|
||||
kp, ki, kd := ctrl.Gains()
|
||||
if kp != 1.0 || ki != 0.5 || kd != 0.1 {
|
||||
t.Errorf("gains not updated correctly: kp=%v, ki=%v, kd=%v", kp, ki, kd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_SetSetpoint(t *testing.T) {
|
||||
ctrl := New(RateLimitWriteTuning())
|
||||
|
||||
ctrl.SetSetpoint(0.7)
|
||||
|
||||
if ctrl.Setpoint() != 0.7 {
|
||||
t.Errorf("setpoint not updated, got %v", ctrl.Setpoint())
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_OutputClamping(t *testing.T) {
|
||||
tuning := pidif.Tuning{
|
||||
Kp: 10.0, // Very high Kp
|
||||
Ki: 0.0,
|
||||
Kd: 0.0,
|
||||
Setpoint: 0.5,
|
||||
DerivativeFilterAlpha: 0.2,
|
||||
IntegralMin: -10,
|
||||
IntegralMax: 10,
|
||||
OutputMin: 0,
|
||||
OutputMax: 1.0, // Strict output max
|
||||
}
|
||||
ctrl := New(tuning)
|
||||
|
||||
// Initialize
|
||||
ctrl.UpdateValue(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Very high error should be clamped
|
||||
out := ctrl.UpdateValue(2.0) // 1.5 error * 10 Kp = 15, should clamp to 1.0
|
||||
if out.Value() > 1.0 {
|
||||
t.Errorf("output should be clamped to 1.0, got %v", out.Value())
|
||||
}
|
||||
if !out.Clamped() {
|
||||
t.Errorf("expected output to be flagged as clamped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_Components(t *testing.T) {
|
||||
tuning := pidif.Tuning{
|
||||
Kp: 1.0,
|
||||
Ki: 0.5,
|
||||
Kd: 0.1,
|
||||
Setpoint: 0.5,
|
||||
DerivativeFilterAlpha: 0.2,
|
||||
IntegralMin: -10,
|
||||
IntegralMax: 10,
|
||||
OutputMin: -100,
|
||||
OutputMax: 100,
|
||||
}
|
||||
ctrl := New(tuning)
|
||||
|
||||
// Initialize
|
||||
ctrl.UpdateValue(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Get components
|
||||
out := ctrl.UpdateValue(0.8)
|
||||
p, i, d := out.Components()
|
||||
|
||||
// Proportional should be positive (0.3 * 1.0 = 0.3)
|
||||
expectedP := 0.3
|
||||
if p < expectedP*0.9 || p > expectedP*1.1 {
|
||||
t.Errorf("expected P term ~%v, got %v", expectedP, p)
|
||||
}
|
||||
|
||||
// Integral should be small but positive (accumulated over ~10ms)
|
||||
if i <= 0 {
|
||||
t.Errorf("expected positive I term, got %v", i)
|
||||
}
|
||||
|
||||
// Derivative should be non-zero (error changed)
|
||||
// The sign depends on filtering and timing
|
||||
_ = d // Just verify it's accessible
|
||||
}
|
||||
|
||||
func TestPresets(t *testing.T) {
|
||||
// Test that all presets create valid controllers
|
||||
tests := []struct {
|
||||
name string
|
||||
tuning pidif.Tuning
|
||||
}{
|
||||
{"RateLimitWrite", RateLimitWriteTuning()},
|
||||
{"RateLimitRead", RateLimitReadTuning()},
|
||||
{"DifficultyAdjustment", DifficultyAdjustmentTuning()},
|
||||
{"TemperatureControl", TemperatureControlTuning(25.0)},
|
||||
{"MotorSpeed", MotorSpeedTuning()},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctrl := New(tt.tuning)
|
||||
if ctrl == nil {
|
||||
t.Error("expected non-nil controller")
|
||||
return
|
||||
}
|
||||
|
||||
// Basic sanity check
|
||||
out := ctrl.UpdateValue(tt.tuning.Setpoint)
|
||||
if out == nil {
|
||||
t.Error("expected non-nil output")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFactoryFunctions(t *testing.T) {
|
||||
// Test convenience factory functions
|
||||
writeCtrl := NewRateLimitWriteController()
|
||||
if writeCtrl == nil {
|
||||
t.Error("NewRateLimitWriteController returned nil")
|
||||
}
|
||||
|
||||
readCtrl := NewRateLimitReadController()
|
||||
if readCtrl == nil {
|
||||
t.Error("NewRateLimitReadController returned nil")
|
||||
}
|
||||
|
||||
diffCtrl := NewDifficultyAdjustmentController()
|
||||
if diffCtrl == nil {
|
||||
t.Error("NewDifficultyAdjustmentController returned nil")
|
||||
}
|
||||
|
||||
tempCtrl := NewTemperatureController(72.0)
|
||||
if tempCtrl == nil {
|
||||
t.Error("NewTemperatureController returned nil")
|
||||
}
|
||||
|
||||
motorCtrl := NewMotorSpeedController()
|
||||
if motorCtrl == nil {
|
||||
t.Error("NewMotorSpeedController returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_ProcessVariableInterface(t *testing.T) {
|
||||
ctrl := New(RateLimitWriteTuning())
|
||||
|
||||
// Test using the full ProcessVariable interface
|
||||
pv := pidif.NewProcessVariableAt(0.9, time.Now())
|
||||
out := ctrl.Update(pv)
|
||||
|
||||
// First call returns 0
|
||||
if out.Value() != 0 {
|
||||
t.Errorf("expected 0 on first call, got %v", out.Value())
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
pv2 := pidif.NewProcessVariableAt(0.95, time.Now())
|
||||
out2 := ctrl.Update(pv2)
|
||||
|
||||
// Above setpoint should produce positive output
|
||||
if out2.Value() <= 0 {
|
||||
t.Errorf("expected positive output above setpoint, got %v", out2.Value())
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_NewWithGains(t *testing.T) {
|
||||
ctrl := NewWithGains(1.0, 0.5, 0.1, 0.7)
|
||||
|
||||
kp, ki, kd := ctrl.Gains()
|
||||
if kp != 1.0 || ki != 0.5 || kd != 0.1 {
|
||||
t.Errorf("gains not set correctly: kp=%v, ki=%v, kd=%v", kp, ki, kd)
|
||||
}
|
||||
|
||||
if ctrl.Setpoint() != 0.7 {
|
||||
t.Errorf("setpoint not set correctly, got %v", ctrl.Setpoint())
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_SetTuning(t *testing.T) {
|
||||
ctrl := NewDefault()
|
||||
|
||||
newTuning := RateLimitWriteTuning()
|
||||
ctrl.SetTuning(newTuning)
|
||||
|
||||
tuning := ctrl.Tuning()
|
||||
if tuning.Kp != newTuning.Kp || tuning.Ki != newTuning.Ki || tuning.Setpoint != newTuning.Setpoint {
|
||||
t.Errorf("tuning not updated correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_SetOutputLimits(t *testing.T) {
|
||||
ctrl := NewDefault()
|
||||
ctrl.SetOutputLimits(-5.0, 5.0)
|
||||
|
||||
tuning := ctrl.Tuning()
|
||||
if tuning.OutputMin != -5.0 || tuning.OutputMax != 5.0 {
|
||||
t.Errorf("output limits not updated: min=%v, max=%v", tuning.OutputMin, tuning.OutputMax)
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_SetIntegralLimits(t *testing.T) {
|
||||
ctrl := NewDefault()
|
||||
ctrl.SetIntegralLimits(-2.0, 2.0)
|
||||
|
||||
tuning := ctrl.Tuning()
|
||||
if tuning.IntegralMin != -2.0 || tuning.IntegralMax != 2.0 {
|
||||
t.Errorf("integral limits not updated: min=%v, max=%v", tuning.IntegralMin, tuning.IntegralMax)
|
||||
}
|
||||
}
|
||||
|
||||
func TestController_SetDerivativeFilter(t *testing.T) {
|
||||
ctrl := NewDefault()
|
||||
ctrl.SetDerivativeFilter(0.5)
|
||||
|
||||
tuning := ctrl.Tuning()
|
||||
if tuning.DerivativeFilterAlpha != 0.5 {
|
||||
t.Errorf("derivative filter alpha not updated: %v", tuning.DerivativeFilterAlpha)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultTuning(t *testing.T) {
|
||||
tuning := pidif.DefaultTuning()
|
||||
|
||||
if tuning.Kp <= 0 || tuning.Ki <= 0 || tuning.Kd <= 0 {
|
||||
t.Error("default tuning should have positive gains")
|
||||
}
|
||||
|
||||
if tuning.DerivativeFilterAlpha <= 0 || tuning.DerivativeFilterAlpha > 1.0 {
|
||||
t.Errorf("default derivative filter alpha should be in (0, 1], got %v", tuning.DerivativeFilterAlpha)
|
||||
}
|
||||
|
||||
if tuning.OutputMin >= tuning.OutputMax {
|
||||
t.Error("default output min should be less than max")
|
||||
}
|
||||
|
||||
if tuning.IntegralMin >= tuning.IntegralMax {
|
||||
t.Error("default integral min should be less than max")
|
||||
}
|
||||
}
|
||||
127
pkg/pid/presets.go
Normal file
127
pkg/pid/presets.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package pid
|
||||
|
||||
import (
|
||||
pidif "next.orly.dev/pkg/interfaces/pid"
|
||||
)
|
||||
|
||||
// Presets for common PID controller use cases.
|
||||
// These provide good starting points that can be fine-tuned for specific applications.
|
||||
|
||||
// RateLimitWriteTuning returns tuning optimized for write rate limiting.
|
||||
// - Aggressive response to prevent memory exhaustion
|
||||
// - Moderate integral for sustained load handling
|
||||
// - Small derivative with strong filtering
|
||||
func RateLimitWriteTuning() pidif.Tuning {
|
||||
return pidif.Tuning{
|
||||
Kp: 0.5,
|
||||
Ki: 0.1,
|
||||
Kd: 0.05,
|
||||
Setpoint: 0.85, // Target 85% of limit
|
||||
DerivativeFilterAlpha: 0.2, // Strong filtering
|
||||
IntegralMin: -2.0,
|
||||
IntegralMax: 10.0,
|
||||
OutputMin: 0.0,
|
||||
OutputMax: 1.0, // Max 1 second delay
|
||||
}
|
||||
}
|
||||
|
||||
// RateLimitReadTuning returns tuning optimized for read rate limiting.
|
||||
// - Less aggressive than writes (reads are more latency-sensitive)
|
||||
// - Lower gains to avoid over-throttling queries
|
||||
func RateLimitReadTuning() pidif.Tuning {
|
||||
return pidif.Tuning{
|
||||
Kp: 0.3,
|
||||
Ki: 0.05,
|
||||
Kd: 0.02,
|
||||
Setpoint: 0.90, // Target 90% of limit
|
||||
DerivativeFilterAlpha: 0.15, // Very strong filtering
|
||||
IntegralMin: -1.0,
|
||||
IntegralMax: 5.0,
|
||||
OutputMin: 0.0,
|
||||
OutputMax: 0.5, // Max 500ms delay
|
||||
}
|
||||
}
|
||||
|
||||
// DifficultyAdjustmentTuning returns tuning for PoW difficulty adjustment.
|
||||
// Designed for block time targeting where:
|
||||
// - Process variable: actual_block_time / target_block_time (1.0 = on target)
|
||||
// - Output: difficulty multiplier (1.0 = no change, >1 = harder, <1 = easier)
|
||||
//
|
||||
// This uses:
|
||||
// - Low Kp to avoid overreacting to individual blocks
|
||||
// - Moderate Ki to converge on target over time
|
||||
// - Small Kd with strong filtering to anticipate trends
|
||||
func DifficultyAdjustmentTuning() pidif.Tuning {
|
||||
return pidif.Tuning{
|
||||
Kp: 0.1, // Low proportional (blocks are noisy)
|
||||
Ki: 0.05, // Moderate integral for convergence
|
||||
Kd: 0.02, // Small derivative
|
||||
Setpoint: 1.0, // Target: actual == expected block time
|
||||
DerivativeFilterAlpha: 0.1, // Very strong filtering (blocks are noisy)
|
||||
IntegralMin: -0.5, // Limit integral windup
|
||||
IntegralMax: 0.5,
|
||||
OutputMin: 0.5, // Min 50% difficulty change
|
||||
OutputMax: 2.0, // Max 200% difficulty change
|
||||
}
|
||||
}
|
||||
|
||||
// TemperatureControlTuning returns tuning for temperature regulation.
|
||||
// Suitable for heating/cooling systems where:
|
||||
// - Process variable: current temperature
|
||||
// - Setpoint: target temperature
|
||||
// - Output: heater/cooler power level (0-1)
|
||||
func TemperatureControlTuning(targetTemp float64) pidif.Tuning {
|
||||
return pidif.Tuning{
|
||||
Kp: 0.1, // Moderate response
|
||||
Ki: 0.01, // Slow integral (thermal inertia)
|
||||
Kd: 0.05, // Some anticipation
|
||||
Setpoint: targetTemp,
|
||||
DerivativeFilterAlpha: 0.3, // Moderate filtering
|
||||
IntegralMin: -100.0,
|
||||
IntegralMax: 100.0,
|
||||
OutputMin: 0.0,
|
||||
OutputMax: 1.0,
|
||||
}
|
||||
}
|
||||
|
||||
// MotorSpeedTuning returns tuning for motor speed control.
|
||||
// - Process variable: actual RPM / target RPM
|
||||
// - Output: motor power level
|
||||
func MotorSpeedTuning() pidif.Tuning {
|
||||
return pidif.Tuning{
|
||||
Kp: 0.5, // Quick response
|
||||
Ki: 0.2, // Eliminate steady-state error
|
||||
Kd: 0.1, // Dampen oscillations
|
||||
Setpoint: 1.0, // Target: actual == desired speed
|
||||
DerivativeFilterAlpha: 0.4, // Moderate filtering
|
||||
IntegralMin: -1.0,
|
||||
IntegralMax: 1.0,
|
||||
OutputMin: 0.0,
|
||||
OutputMax: 1.0,
|
||||
}
|
||||
}
|
||||
|
||||
// NewRateLimitWriteController creates a controller for write rate limiting.
|
||||
func NewRateLimitWriteController() *Controller {
|
||||
return New(RateLimitWriteTuning())
|
||||
}
|
||||
|
||||
// NewRateLimitReadController creates a controller for read rate limiting.
|
||||
func NewRateLimitReadController() *Controller {
|
||||
return New(RateLimitReadTuning())
|
||||
}
|
||||
|
||||
// NewDifficultyAdjustmentController creates a controller for PoW difficulty.
|
||||
func NewDifficultyAdjustmentController() *Controller {
|
||||
return New(DifficultyAdjustmentTuning())
|
||||
}
|
||||
|
||||
// NewTemperatureController creates a controller for temperature regulation.
|
||||
func NewTemperatureController(targetTemp float64) *Controller {
|
||||
return New(TemperatureControlTuning(targetTemp))
|
||||
}
|
||||
|
||||
// NewMotorSpeedController creates a controller for motor speed control.
|
||||
func NewMotorSpeedController() *Controller {
|
||||
return New(MotorSpeedTuning())
|
||||
}
|
||||
Submodule pkg/protocol/blossom/blossom deleted from e8d0a1ec44
24
pkg/protocol/blossom/blossom/LICENSE.txt
Normal file
24
pkg/protocol/blossom/blossom/LICENSE.txt
Normal file
@@ -0,0 +1,24 @@
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <https://unlicense.org>
|
||||
61
pkg/protocol/blossom/blossom/README.md
Normal file
61
pkg/protocol/blossom/blossom/README.md
Normal file
@@ -0,0 +1,61 @@
|
||||
# 🌸 Blossom - Blobs stored simply on mediaservers
|
||||
|
||||
Blossom uses [nostr](https://github.com/nostr-protocol/nostr) public / private keys for identities. Users are expected to sign authorization events to prove their identity when interacting with servers
|
||||
|
||||
## What is it?
|
||||
|
||||
Blossom is a specification for a set of HTTP endpoints that allow users to store blobs of data on publicly accessible servers
|
||||
|
||||
## What are blobs
|
||||
|
||||
Blobs are packs of binary data addressed by their sha256 hash
|
||||
|
||||
## Protocol specification (BUDs)
|
||||
|
||||
BUDs or **Blossom Upgrade Documents** are short documents that outline an additional feature that a blossom server may implement.
|
||||
|
||||
## BUDs
|
||||
|
||||
- [BUD-00: Blossom Upgrade Documents](./buds/00.md)
|
||||
- [BUD-01: Server requirements and blob retrieval](./buds/01.md)
|
||||
- [BUD-02: Blob upload and management](./buds/02.md)
|
||||
- [BUD-03: User Server List](./buds/03.md)
|
||||
- [BUD-04: Mirroring blobs](./buds/04.md)
|
||||
- [BUD-05: Media optimization](./buds/05.md)
|
||||
- [BUD-06: Upload requirements](./buds/06.md)
|
||||
- [BUD-07: Payment required](./buds/07.md)
|
||||
- [BUD-08: Nostr File Metadata Tags](./buds/08.md)
|
||||
- [BUD-09: Blob Report](./buds/09.md)
|
||||
|
||||
## Endpoints
|
||||
|
||||
Blossom Servers expose a few endpoints for managing blobs
|
||||
|
||||
- `GET /<sha256>` (optional file `.ext`) [BUD-01](./buds/01.md#get-sha256---get-blob)
|
||||
- `HEAD /<sha256>` (optional file `.ext`) [BUD-01](./buds/01.md#head-sha256---has-blob)
|
||||
- `PUT /upload` [BUD-02](./buds/02.md#put-upload---upload-blob)
|
||||
- `Authentication`: Signed [nostr event](./buds/02.md#upload-authorization-required)
|
||||
- Return a blob descriptor
|
||||
- `HEAD /upload` [BUD-06](./buds/06.md#head-upload---upload-requirements)
|
||||
- `GET /list/<pubkey>` [BUD-02](./buds/02.md#get-listpubkey---list-blobs)
|
||||
- Returns an array of blob descriptors
|
||||
- `Authentication` _(optional)_: Signed [nostr event](./buds/02.md#list-authorization-optional)
|
||||
- `DELETE /<sha256>` [BUD-02](./buds/02.md#delete-sha256---delete-blob)
|
||||
- `Authentication`: Signed [nostr event](./buds/02.md#delete-authorization-required)
|
||||
- `PUT /mirror` [BUD-04](./buds/04.md#put-mirror---mirror-blob)
|
||||
- `Authentication`: Signed [nostr event](./buds/02.md#upload-authorization-required)
|
||||
- `HEAD /media` [BUD-05](./buds/05.md#head-media)
|
||||
- `PUT /media` [BUD-05](./buds/05.md#put-media)
|
||||
- `Authentication`: Signed [nostr event](./buds/05.md#upload-authorization)
|
||||
- `PUT /report` [BUD-09](./buds/09.md)
|
||||
|
||||
## Event kinds
|
||||
|
||||
| kind | description | BUD |
|
||||
| ------- | ------------------- | ------------------ |
|
||||
| `24242` | Authorization event | [01](./buds/01.md) |
|
||||
| `10063` | User Server List | [03](./buds/03.md) |
|
||||
|
||||
## License
|
||||
|
||||
Public domain.
|
||||
19
pkg/protocol/blossom/blossom/buds/00.md
Normal file
19
pkg/protocol/blossom/blossom/buds/00.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# BUD-00
|
||||
|
||||
## Blossom Upgrade Documents
|
||||
|
||||
`draft` `mandatory`
|
||||
|
||||
This document details the common language for all following BUDs
|
||||
|
||||
## Language
|
||||
|
||||
All occurences of "MUST", "MUST NOT", "SHOULD", "SHOULD NOT" MUST be interpreted as per [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119)
|
||||
|
||||
## BUDs
|
||||
|
||||
BUDs or "Blossom Upgrade Documents" are short documents that outline an additional requirement or feature that a blossom server MUST or MAY implement.
|
||||
|
||||
## Blobs
|
||||
|
||||
Blobs are raw binary data addressed by the sha256 hash of the data.
|
||||
162
pkg/protocol/blossom/blossom/buds/01.md
Normal file
162
pkg/protocol/blossom/blossom/buds/01.md
Normal file
@@ -0,0 +1,162 @@
|
||||
# BUD-01
|
||||
|
||||
## Server requirements and blob retrieval
|
||||
|
||||
`draft` `mandatory`
|
||||
|
||||
_All pubkeys MUST be in hex format_
|
||||
|
||||
## Cross origin headers
|
||||
|
||||
Servers MUST set the `Access-Control-Allow-Origin: *` header on all responses to ensure compatibility with applications hosted on other domains.
|
||||
|
||||
For [preflight](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS#preflighted_requests) (`OPTIONS`) requests,
|
||||
servers MUST also set, at minimum, the `Access-Control-Allow-Headers: Authorization, *` and `Access-Control-Allow-Methods: GET, HEAD, PUT,
|
||||
DELETE` headers.
|
||||
|
||||
The header `Access-Control-Max-Age: 86400` MAY be set to cache the results of a preflight request for 24 hours.
|
||||
|
||||
## Error responses
|
||||
|
||||
Every time a server sends an error response (HTTP status codes >=400), it may include a human-readable header `X-Reason` that can be displayed to the user.
|
||||
|
||||
## Authorization events
|
||||
|
||||
Authorization events are used to identify the users to the server
|
||||
|
||||
Authorization events must be generic and must NOT be scoped to specific servers. This allows pubkeys to sign a single event and interact the same way with multiple servers.
|
||||
|
||||
Events MUST be kind `24242` and have a `t` tag with a verb of `get`, `upload`, `list`, or `delete`
|
||||
|
||||
Events MUST have the `content` set to a human readable string explaining to the user what the events intended use is. For example `Upload Blob`, `Delete dog-picture.png`, `List Images`, etc
|
||||
|
||||
All events MUST have a [NIP-40](https://github.com/nostr-protocol/nips/blob/master/40.md) `expiration` tag set to a unix timestamp at which the event should be considered expired.
|
||||
|
||||
Authorization events MAY have multiple `x` tags for endpoints that require a sha256 hash.
|
||||
|
||||
Example event:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": "bb653c815da18c089f3124b41c4b5ec072a40b87ca0f50bbbc6ecde9aca442eb",
|
||||
"pubkey": "b53185b9f27962ebdf76b8a9b0a84cd8b27f9f3d4abd59f715788a3bf9e7f75e",
|
||||
"kind": 24242,
|
||||
"content": "Upload bitcoin.pdf",
|
||||
"created_at": 1708773959,
|
||||
"tags": [
|
||||
["t", "upload"],
|
||||
// Authorization events MAY have multiple "x" tags.
|
||||
["x", "b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553"],
|
||||
["expiration", "1708858680"]
|
||||
],
|
||||
"sig": "d0d58c92afb3f4f1925120b99c39cffe77d93e82f488c5f8f482e8f97df75c5357175b5098c338661c37d1074b0a18ab5e75a9df08967bfb200930ec6a76562f"
|
||||
}
|
||||
```
|
||||
|
||||
Servers must perform the following checks in order to validate the event
|
||||
|
||||
1. The `kind` must be `24242`
|
||||
2. `created_at` must be in the past
|
||||
3. The `expiration` tag must be set to a Unix timestamp in the future
|
||||
4. The `t` tag must have a verb matching the intended action of the endpoint
|
||||
5. Additional checks for specific endpoints. `/upload`, `/delete`, etc
|
||||
|
||||
Using the `Authorization` HTTP header, the kind `24242` event MUST be base64 encoded and use the Authorization scheme Nostr
|
||||
|
||||
Example HTTP Authorization header:
|
||||
|
||||
```
|
||||
Authorization: Nostr eyJpZCI6IjhlY2JkY2RkNTMyOTIwMDEwNTUyNGExNDI4NzkxMzg4MWIzOWQxNDA5ZDhiOTBjY2RiNGI0M2Y4ZjBmYzlkMGMiLCJwdWJrZXkiOiI5ZjBjYzE3MDIzYjJjZjUwOWUwZjFkMzA1NzkzZDIwZTdjNzIyNzY5MjhmZDliZjg1NTM2ODg3YWM1NzBhMjgwIiwiY3JlYXRlZF9hdCI6MTcwODc3MTIyNywia2luZCI6MjQyNDIsInRhZ3MiOltbInQiLCJnZXQiXSxbImV4cGlyYXRpb24iLCIxNzA4ODU3NTQwIl1dLCJjb250ZW50IjoiR2V0IEJsb2JzIiwic2lnIjoiMDJmMGQyYWIyM2IwNDQ0NjI4NGIwNzFhOTVjOThjNjE2YjVlOGM3NWFmMDY2N2Y5NmNlMmIzMWM1M2UwN2I0MjFmOGVmYWRhYzZkOTBiYTc1NTFlMzA4NWJhN2M0ZjU2NzRmZWJkMTVlYjQ4NTFjZTM5MGI4MzI4MjJiNDcwZDIifQ==
|
||||
```
|
||||
|
||||
## Endpoints
|
||||
|
||||
All endpoints MUST be served from the root of the domain (eg. the `/upload` endpoint MUST be accessible from `https://cdn.example.com/upload`, etc). This allows clients to talk to servers interchangeably when uploading or retrieving blobs
|
||||
|
||||
## GET /sha256 - Get Blob
|
||||
|
||||
The `GET /<sha256>` endpoint MUST return the contents of the blob in the response body. the `Content-Type` header SHOULD beset to the appropriate MIME-type
|
||||
|
||||
The endpoint MUST accept an optional file extension in the URL. ie. `.pdf`, `.png`, etc
|
||||
|
||||
Regardless of the file extension, the server MUST return the MIME type of the blob in the `Content-Type` header. If the
|
||||
server does not know the MIME type of the blob, it MUST default to `application/octet-stream`
|
||||
|
||||
### Proxying and Redirection (Optional)
|
||||
|
||||
If the endpoint returns a redirection 3xx status code such as 307 or 308 ([RFC 9110 section
|
||||
15.4](https://datatracker.ietf.org/doc/html/rfc9110#name-redirection-3xx)), it MUST redirect to a URL containing the
|
||||
same sha256 hash as the requested blob. This ensures that if a user copies or reuses the redirect URL, it will
|
||||
contain the original sha256 hash.
|
||||
|
||||
While the final blob may not be served from a Blossom server (e.g. CDN, IPFS, object storage, etc.), the destination
|
||||
server MUST set the `Access-Control-Allow-Origin: *` header on the response to allow cross-origin requests, as well as
|
||||
the `Content-Type` and `Content-Length` headers to ensure the blob can be correctly displayed by clients. Two ways to
|
||||
guarantee this are:
|
||||
|
||||
1. Proxying the blob through the Blossom server, allowing it to override headers such as `Content-Type`.
|
||||
2. Manipulating the redirect URL to include a file extension that matches the blob type, such as `.pdf`, `.png`, etc. If
|
||||
the server is unable to determine the MIME type of the blob, it MUST default to `application/octet-stream` and MAY
|
||||
include a file extension in the URL that reflects the blob type (e.g. `.bin`, `.dat`, etc.).
|
||||
|
||||
### Get Authorization (optional)
|
||||
|
||||
The server may optionally require authorization when retrieving blobs from the `GET /<sha256>` endpoint
|
||||
|
||||
In this case, the server MUST perform additional checks on the authorization event
|
||||
|
||||
1. A `t` tag MUST be present and set to `get`
|
||||
2. The event MUST contain either a `server` tag containing the full URL to the server or MUST contain at least one `x` tag matching the sha256 hash of the blob being retrieved
|
||||
|
||||
If the client did not send an `Authorization` header the server must respond with the appropriate HTTP status code `401` (Unauthorized)
|
||||
|
||||
Example event for retrieving a single blob:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "06d4842b9d7f8bf72440471704de4efa9ef8f0348e366d097405573994f66294",
|
||||
"pubkey": "ec0d11351457798907a3900fe465bfdc3b081be6efeb3d68c4d67774c0bc1f9a",
|
||||
"kind": 24242,
|
||||
"content": "Get bitcoin.pdf",
|
||||
"created_at": 1708771927,
|
||||
"tags": [
|
||||
["t", "get"],
|
||||
["expiration", "1708857340"],
|
||||
["x", "b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553"]
|
||||
],
|
||||
"sig": "22ecb5116ba143e4c3d6dc4b53d549aed6970ec455f6d25d145e0ad1fd7c0e26c465b2e92d5fdf699c7050fa43e6a41f087ef167208d4f06425f61548168fd7f"
|
||||
}
|
||||
```
|
||||
|
||||
Example event for retrieving multiple blobs from single server:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "d9484f18533d5e36f000f902a45b15a7eecf5fbfcb046789756d57ea87115dc5",
|
||||
"pubkey": "b5f07faa8d3529f03bd898a23dfb3257bab8d8f5490777c46076ff9647e205dc",
|
||||
"kind": 24242,
|
||||
"content": "Get blobs from example.com",
|
||||
"created_at": 1708771927,
|
||||
"tags": [
|
||||
["t", "get"],
|
||||
["expiration", "1708857340"],
|
||||
["server", "https://cdn.example.com/"]
|
||||
],
|
||||
"sig": "e402ade78e1714d40cd6bd3091bc5f4ada8e904e90301b5a2b9b5f0b6e95ce908d4f22b15e9fb86f8268a2131f8adbb3d1f0e7e7afd1ab0f4f08acb15822a999"
|
||||
}
|
||||
```
|
||||
|
||||
## HEAD /sha256 - Has Blob
|
||||
|
||||
The `HEAD /<sha256>` endpoint SHOULD be identical to the `GET /<sha256>` endpoint except that it MUST NOT return the
|
||||
blob in the reponse body per [RFC 7231](https://www.rfc-editor.org/rfc/rfc7231#section-4.3.2)
|
||||
|
||||
The endpoint MUST respond with the same `Content-Type` and `Content-Length` headers as the `GET /<sha256>` endpoint.
|
||||
|
||||
The endpoint MUST accept an optional file extension in the URL similar to the `GET /<sha256>` endpoint. ie. `.pdf`, `.png`, etc
|
||||
|
||||
## Range requests
|
||||
|
||||
To better support mobile devices, video files, or low bandwidth connections. servers should support range requests ([RFC 7233 section 3](https://www.rfc-editor.org/rfc/rfc7233#section-3)) on the `GET /<sha256>` endpoint and signal support using the `accept-ranges: bytes` and `content-length` headers on the `HEAD /<sha256>` endpoint
|
||||
|
||||
See [MDN docs](https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests) for more details
|
||||
148
pkg/protocol/blossom/blossom/buds/02.md
Normal file
148
pkg/protocol/blossom/blossom/buds/02.md
Normal file
@@ -0,0 +1,148 @@
|
||||
# BUD-02
|
||||
|
||||
## Blob upload and management
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
_All pubkeys MUST be in hex format_
|
||||
|
||||
Defines the `/upload`, `/list` and `DELETE /<sha256>` endpoints
|
||||
|
||||
## Blob Descriptor
|
||||
|
||||
A blob descriptor is a JSON object containing `url`, `sha256`, `size`, `type`, and `uploaded` fields
|
||||
|
||||
- `url` A publicly accessible URL to the [BUD-01](./01.md#get-sha256---get-blob) `GET /<sha256>` endpoint with a file extension
|
||||
- `sha256` The sha256 hash of the blob
|
||||
- `size` The size of the blob in bytes
|
||||
- `type` The MIME type of the blob (falling back to `application/octet-stream` if unknown)
|
||||
- `uploaded` The unix timestamp of when the blob was uploaded to the server
|
||||
|
||||
Servers MUST include a file extension in the URL in the `url` field to allow clients to easily embed the URL in social posts or other content
|
||||
|
||||
Servers MAY include additional fields in the descriptor like `magnet`, `infohash`, or `ipfs` depending on other protocols they support
|
||||
|
||||
Example:
|
||||
|
||||
```json
|
||||
{
|
||||
"url": "https://cdn.example.com/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf",
|
||||
"sha256": "b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553",
|
||||
"size": 184292,
|
||||
"type": "application/pdf",
|
||||
"uploaded": 1725105921
|
||||
}
|
||||
```
|
||||
|
||||
## PUT /upload - Upload Blob
|
||||
|
||||
The `PUT /upload` endpoint MUST accept binary data in the body of the request and MAY use the `Content-Type` and `Content-Length` headers to get the MIME type and size of the data
|
||||
|
||||
The endpoint MUST NOT modify the blob in any way and should return the exact same sha256 that was uploaded. This is critical to allow users to re-upload their blobs to new servers
|
||||
|
||||
The endpoint MUST return a [Blob Descriptor](#blob-descriptor) if the upload was successful or an error object if it was not
|
||||
|
||||
Servers MAY reject an upload for any reason and should respond with the appropriate HTTP `4xx` status code and an error
|
||||
message explaining the reason for the rejection
|
||||
|
||||
### File extension normalization (Optional)
|
||||
|
||||
When storing blobs, servers MAY normalise the file extension to a standard format (e.g. `.pdf`, `.png`, etc.) based on
|
||||
the MIME type of the blob. This can be especially useful when the `GET /<sha256>` endpoint is redirected to an external
|
||||
URL (see the [proxying and redirection section from BUD-01](./01.md#proxying-and-redirection-optional)), as external
|
||||
servers may rely on the file extension to serve the blob correctly.
|
||||
|
||||
### Upload Authorization (Optional)
|
||||
|
||||
Servers MAY accept an authorization event when uploading blobs and should perform additional checks
|
||||
|
||||
1. The `t` tag MUST be set to `upload`
|
||||
2. MUST contain at least one `x` tag matching the sha256 hash of the body of the request
|
||||
|
||||
Example Authorization event:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "bb653c815da18c089f3124b41c4b5ec072a40b87ca0f50bbbc6ecde9aca442eb",
|
||||
"pubkey": "b53185b9f27962ebdf76b8a9b0a84cd8b27f9f3d4abd59f715788a3bf9e7f75e",
|
||||
"kind": 24242,
|
||||
"content": "Upload bitcoin.pdf",
|
||||
"created_at": 1708773959,
|
||||
"tags": [
|
||||
["t", "upload"],
|
||||
["x", "b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553"],
|
||||
["expiration", "1708858680"]
|
||||
],
|
||||
"sig": "d0d58c92afb3f4f1925120b99c39cffe77d93e82f488c5f8f482e8f97df75c5357175b5098c338661c37d1074b0a18ab5e75a9df08967bfb200930ec6a76562f"
|
||||
}
|
||||
```
|
||||
|
||||
## GET /list/pubkey - List Blobs (Optional)
|
||||
|
||||
The `/list/<pubkey>` endpoint MUST return a JSON array of [Blob Descriptor](#blob-descriptor) that were uploaded by the specified pubkey
|
||||
|
||||
The endpoint MUST support a `since` and `until` query parameter to limit the returned blobs by their `uploaded` date
|
||||
|
||||
Servers may reject a list for any reason and MUST respond with the appropriate HTTP `4xx` status code and an error message explaining the reason for the rejection
|
||||
|
||||
### List Authorization (optional)
|
||||
|
||||
The server may optionally require Authorization when listing blobs uploaded by the pubkey
|
||||
|
||||
In this case the server must perform additional checks on the authorization event
|
||||
|
||||
1. The `t` tag must be set to `list`
|
||||
|
||||
Example Authorization event:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "cbb1cab9566355bfdf04e1f1fc1e655fe903ecc193e8a750092ee53beec2a0e8",
|
||||
"pubkey": "a5fc3654296e6de3cda6ba3e8eba7224fac8b150fd035d66b4c3c1dc2888b8fc",
|
||||
"kind": 24242,
|
||||
"content": "List Blobs",
|
||||
"created_at": 1708772350,
|
||||
"tags": [
|
||||
["t", "list"],
|
||||
["expiration", "1708858680"]
|
||||
],
|
||||
"sig": "ff9c716f8de0f633738036472be553ce4b58dc71d423a0ef403f95f64ef28582ef82129b41d4d0ef64d2338eb4aeeb66dbc03f8b3a3ed405054ea8ecb14fa36c"
|
||||
}
|
||||
```
|
||||
|
||||
## DELETE /sha256 - Delete Blob
|
||||
|
||||
Servers MUST accept `DELETE` requests to the `/<sha256>` endpoint
|
||||
|
||||
Servers may reject a delete request for any reason and should respond with the appropriate HTTP `4xx` status code and an error message explaining the reason for the rejection
|
||||
|
||||
### Delete Authorization (required)
|
||||
|
||||
Servers MUST accept an authorization event when deleting blobs
|
||||
|
||||
Servers should perform additional checks on the authorization event
|
||||
|
||||
1. The `t` tag must be set to `delete`
|
||||
2. MUST contain at least one `x` tag matching the sha256 hash of the blob being deleted
|
||||
|
||||
When multiple `x` tags are present on the authorization event the server MUST only delete the blob listed in the URL.
|
||||
|
||||
**Multiple `x` tags MUST NOT be interpreted as the user requesting a bulk delete.**
|
||||
|
||||
Example Authorization event:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "a92868bd8ea740706d931f5d205308eaa0e6698e5f8026a990e78ee34ce47fe8",
|
||||
"pubkey": "ae0063dd2c81ec469f2291ac029a19f39268bfc40aea7ab4136d7a858c3a06de",
|
||||
"kind": 24242,
|
||||
"content": "Delete bitcoin.pdf",
|
||||
"created_at": 1708774469,
|
||||
"tags": [
|
||||
["t", "delete"],
|
||||
["x", "b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553"],
|
||||
["expiration", "1708858680"]
|
||||
],
|
||||
"sig": "2ba9af680505583e3eb289a1624a08661a2f6fa2e5566a5ee0036333d517f965e0ffba7f5f7a57c2de37e00a2e85fd7999076468e52bdbcfad8abb76b37a94b0"
|
||||
}
|
||||
```
|
||||
76
pkg/protocol/blossom/blossom/buds/03.md
Normal file
76
pkg/protocol/blossom/blossom/buds/03.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# BUD-03
|
||||
|
||||
## User Server List
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
Defines a replaceable event using `kind:10063` to advertise the blossom servers a user uses to host their blobs.
|
||||
|
||||
The event MUST include at least one `server` tag containing the full server URL including the `http://` or `https://`.
|
||||
|
||||
The order of these tags is important and should be arranged with the users most "reliable" or "trusted" servers being first.
|
||||
|
||||
The `.content` field is not used.
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "e4bee088334cb5d38cff1616e964369c37b6081be997962ab289d6c671975d71",
|
||||
"pubkey": "781208004e09102d7da3b7345e64fd193cd1bc3fce8fdae6008d77f9cabcd036",
|
||||
"content": "",
|
||||
"kind": 10063,
|
||||
"created_at": 1708774162,
|
||||
"tags": [
|
||||
["server", "https://cdn.self.hosted"],
|
||||
["server", "https://cdn.satellite.earth"]
|
||||
],
|
||||
"sig": "cc5efa74f59e80622c77cacf4dd62076bcb7581b45e9acff471e7963a1f4d8b3406adab5ee1ac9673487480e57d20e523428e60ffcc7e7a904ac882cfccfc653"
|
||||
}
|
||||
```
|
||||
|
||||
## Client Upload Implementation
|
||||
|
||||
When uploading blobs clients MUST attempt to upload the blob to at least the first `server` listed in the users server list.
|
||||
|
||||
Optionally clients MAY upload the blob to all the servers or mirror the blob to the other servers if they support [BUD-04](./04.md)
|
||||
|
||||
This ensures that the blob is available in multiple locations in the case one of the servers goes offline.
|
||||
|
||||
## Client Retrieval Implementation
|
||||
|
||||
When extracting the SHA256 hash from the URL clients MUST use the last occurrence of a 64 char hex string. This allows clients to extract hashes from blossom URLs and SOME non-blossom URLs.
|
||||
|
||||
In all the following examples, the hash `b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553` should be selected
|
||||
|
||||
- Blossom URLs
|
||||
- `https://blossom.example.com/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf`
|
||||
- `https://cdn.example.com/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553`
|
||||
- Non Blossom URLs
|
||||
- `https://cdn.example.com/user/ec4425ff5e9446080d2f70440188e3ca5d6da8713db7bdeef73d0ed54d9093f0/media/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf`
|
||||
- `https://cdn.example.com/media/user-name/documents/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf`
|
||||
- `http://download.example.com/downloads/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553`
|
||||
- `http://media.example.com/documents/b1/67/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf`
|
||||
|
||||
In the context of nostr events, clients SHOULD use the author's server list when looking for blobs that are no longer available at the original URL.
|
||||
|
||||
Take the following event as an example
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "834185269f4ab72539193105060dbb1c8b2efd702d14481cea345c47beefe6eb",
|
||||
"pubkey": "ec4425ff5e9446080d2f70440188e3ca5d6da8713db7bdeef73d0ed54d9093f0",
|
||||
"content": "I've developed a new open source P2P e-cash system called Bitcoin. check it out\nhttps://cdn.broken-domain.com/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf",
|
||||
"kind": 1,
|
||||
"created_at": 1297484820,
|
||||
"tags": [],
|
||||
"sig": "bd4bb200bdd5f7ffe5dbc3e539052e27b05d6f9f528e255b1bc4261cc16b8f2ad85c89eef990c5f2eee756ef71b4c571ecf6a88ad12f7338e321dd60c6a903b5"
|
||||
}
|
||||
```
|
||||
|
||||
Once the client discovers that the URL `https://cdn.broken-domain.com/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf` is no longer available. It can perform the following steps to find the blob:
|
||||
|
||||
1. Get the SHA256 hash from the URL
|
||||
2. Look for the authors server list `kind:10063`
|
||||
3. If found, Attempt to retrieve the blob from each `server` listed started with the first
|
||||
4. If not found, the client MAY fallback to using a well-known popular blossom server to retrieve the blob
|
||||
|
||||
This ensures clients can quickly find missing blobs using the users list of trusted servers.
|
||||
46
pkg/protocol/blossom/blossom/buds/04.md
Normal file
46
pkg/protocol/blossom/blossom/buds/04.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# BUD-04
|
||||
|
||||
## Mirroring blobs
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
Defines the `/mirror` endpoint
|
||||
|
||||
## PUT /mirror - Mirror Blob
|
||||
|
||||
A server MAY expose a `PUT /mirror` endpoint to allow users to copy a blob from a URL instead of uploading it
|
||||
|
||||
Clients MUST pass the URL of the remote blob as a stringified JSON object in the request body
|
||||
|
||||
```jsonc
|
||||
// request body...
|
||||
{
|
||||
"url": "https://cdn.satellite.earth/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf"
|
||||
}
|
||||
```
|
||||
|
||||
Clients MAY set the `Authorization` header to an upload authorization event defined in [BUD-02](./02.md#upload-authorization-optional). When using authorization, the event MUST be of type "upload".
|
||||
|
||||
The `/mirror` endpoint MUST download the blob from the specified URL and verify that there is at least one `x` tag in the authorization event matching the sha256 hash of the download blob
|
||||
|
||||
**Multiple `x` tags in the authorization event MUST NOT be interpreted as the user requesting to mirror multiple blobs.**
|
||||
|
||||
The endpoint MUST return a [Blob Descriptor](#blob-descriptor) and a `2xx` status code if the mirroring was successful
|
||||
or a `4xx` status code and error message if it was not.
|
||||
|
||||
The destination server SHOULD use the `Content-Type` header returned from the origin server to infer the mime type of
|
||||
the blob. If the `Content-Type` header is not present the destination server SHOULD attempt to detect the `Content-Type`
|
||||
from the blob contents and file extension, falling back to `application/octet-stream` if it cannot determine the type.
|
||||
|
||||
Servers MAY use the `Content-Length` header to determine the size of the blob.
|
||||
|
||||
Servers MAY reject a mirror request for any reason and MUST respond with the appropriate HTTP `4xx` status code and an error message explaining the reason for the rejection.
|
||||
|
||||
## Example Flow
|
||||
|
||||
1. Client signs an `upload` authorization event and uploads blob to Server A
|
||||
1. Server A returns a [Blob Descriptor](./02.md#blob-descriptor) with the `url`
|
||||
1. Client sends the `url` to Server B `/mirror` using the original `upload` authorization event
|
||||
1. Server B downloads the blob from Server A using the `url`
|
||||
1. Server B verifies the downloaded blob hash matches the `x` tag in the authorization event
|
||||
1. Server B returns a [Blob Descriptor](./02.md#blob-descriptor)
|
||||
48
pkg/protocol/blossom/blossom/buds/05.md
Normal file
48
pkg/protocol/blossom/blossom/buds/05.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# BUD-05
|
||||
|
||||
## Media optimization endpoints
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
Defines the `PUT /media` endpoint for processing and optimizing media
|
||||
|
||||
## PUT /media
|
||||
|
||||
The `PUT /media` endpoint MUST accept binary data in the body of the request and MAY use the `Content-Type` and `Content-Length` headers to get the MIME type and size of the media
|
||||
|
||||
The server should preform any optimizations or conversions it deems necessary in order to make the media more suitable for distribution
|
||||
|
||||
The endpoint MUST respond with a `2xx` status and a [blob descriptor](./02.md#blob-descriptor) of the new processed blob
|
||||
|
||||
Servers MAY reject media uploads for any reason and should respond with the appropriate HTTP `4xx` status code and an error message explaining the reason for the rejection
|
||||
|
||||
### Upload Authorization
|
||||
|
||||
Servers MAY require a `media` [authorization event](./02.md#upload-authorization-required) to identify the uploader
|
||||
|
||||
If a server requires a `media` authorization event it MUST perform the following checks
|
||||
|
||||
1. The `t` tag MUST be set to `media`
|
||||
2. MUST contain at least one `x` tag matching the sha256 hash of the body of the request
|
||||
|
||||
## HEAD /media
|
||||
|
||||
Servers MUST respond to `HEAD` requests on the `/media` endpoint in a similar way to the `HEAD /upload` endpoint defined in [BUD-06](./06.md)
|
||||
|
||||
## Limitations
|
||||
|
||||
This endpoint is intentionally limited to optimizing a single blob with the goal of making it easier to distribute
|
||||
|
||||
How the blob is optimized is the sole responsibility of the server and the client should have no say in what optimization process is used
|
||||
|
||||
The goal of this endpoint is to provide a simple "trusted" optimization endpoint clients can use to optimize media for distribution
|
||||
|
||||
If a longer optimization or transformation process is needed, or if the client needs to specify how a blob should be transformed. there are other tools and protocol that should be used.
|
||||
|
||||
## Client Implementation
|
||||
|
||||
Clients MAY let a user selected a "trusted processing" server for uploading images or short videos
|
||||
|
||||
Once a server has been selected, the client uploads the original media to the `/media` endpoint of the trusted server and get the optimized blob back
|
||||
|
||||
Then the client can ask the user to sign another `upload` authorization event for the new optimized blob and call the `/mirror` endpoint on other servers to distribute the blob
|
||||
73
pkg/protocol/blossom/blossom/buds/06.md
Normal file
73
pkg/protocol/blossom/blossom/buds/06.md
Normal file
@@ -0,0 +1,73 @@
|
||||
# BUD-06
|
||||
|
||||
## Upload requirements
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
Defines how clients can verify if the upload can be completed before sending the blob to the server. This mechanism helps prevent unnecessary traffic to other endpoints by rejecting files based on their hash, size, MIME type or other server-specific requirements.
|
||||
|
||||
## HEAD /upload - Upload requirements
|
||||
|
||||
The `HEAD /upload` endpoint MUST use the `X-SHA-256`, `X-Content-Type` and `X-Content-Length` headers sent by client to get the SHA-256 hash, MIME type and size of the blob that will be uploaded, returning a HTTP status code and a custom header `X-Reason` to indicate some human readable message about the upload requirements.
|
||||
|
||||
### Headers
|
||||
|
||||
- `X-SHA-256`: A string that represents the blob's SHA-256 hash.
|
||||
- `X-Content-Length`: An integer that represents the blob size in bytes.
|
||||
- `X-Content-Type`: A string that specifies the blob's MIME type, like `application/pdf` or `image/png`.
|
||||
|
||||
### Upload Authorization
|
||||
|
||||
The `HEAD /upload` endpoint MAY accept an `upload` authorization event using the `Authorization` header similar to what is used in the [`PUT /upload`](./02.md#upload-authorization-required) endpoint
|
||||
|
||||
If the server requires authorization to upload it may respond with the `401` status code, or if authorization was provided and is invalid or not permitted it may respond with `403` status code
|
||||
|
||||
### Examples
|
||||
|
||||
Example request from the client:
|
||||
|
||||
```http
|
||||
X-Content-Type: application/pdf
|
||||
X-Content-Length: 184292
|
||||
X-SHA-256: 88a74d0b866c8ba79251a11fe5ac807839226870e77355f02eaf68b156522576
|
||||
```
|
||||
|
||||
Example response from the server if the upload can be done:
|
||||
|
||||
```http
|
||||
HTTP/1.1 200 OK
|
||||
```
|
||||
|
||||
If the upload cannot proceed, the server MUST return an appropriate `4xx` HTTP status code and a custom header `X-Reason` with a human readable error message.
|
||||
|
||||
Some examples of error messages:
|
||||
|
||||
```http
|
||||
HTTP/1.1 400 Bad Request
|
||||
X-Reason: Invalid X-SHA-256 header format. Expected a string.
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 401 Unauthorized
|
||||
X-Reason: Authorization required for uploading video files.
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 403 Forbidden
|
||||
X-Reason: SHA-256 hash banned.
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 411 Length Required
|
||||
X-Reason: Missing X-Content-Length header.
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 413 Content Too Large
|
||||
X-Reason: File too large. Max allowed size is 100MB.
|
||||
```
|
||||
|
||||
```http
|
||||
HTTP/1.1 415 Unsupported Media Type
|
||||
X-Reason: Unsupported file type.
|
||||
```
|
||||
105
pkg/protocol/blossom/blossom/buds/07.md
Normal file
105
pkg/protocol/blossom/blossom/buds/07.md
Normal file
@@ -0,0 +1,105 @@
|
||||
BUD-07
|
||||
======
|
||||
|
||||
Paid upload and download
|
||||
---------------
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
Payment requirements for blob storage.
|
||||
|
||||
## Payment Required
|
||||
|
||||
Some servers MAY require payment for uploads, downloads, or any other endpoint. In such cases, these endpoints MUST return a **402 Payment Required** status code.
|
||||
|
||||
Some endpoints a server may require payment for:
|
||||
|
||||
- [`HEAD /upload`](./06.md) to signal that payment is required for the `PUT` request ( if [BUD-06](./06.md) is supported )
|
||||
- [`PUT /upload`](./02.md#put-upload---upload-blob) to require payment for uploads
|
||||
- [`HEAD /<sha256>`](./01.md#head-sha256---has-blob) to signal that payment is required for the `GET` request
|
||||
- [`GET /<sha256>`](./01.md#get-sha256---get-blob) to require payment for downloads ( maybe charge by MB downloaded? )
|
||||
- [`HEAD /media`](./05.md) and [`PUT /upload`](./05.md) to require payment for media optimizations ( if [BUD-06](./06.md) is supported )
|
||||
|
||||
When payment is required, the server MUST include one or more `X-{payment_method}` header(s), each corresponding to a supported payment method.
|
||||
|
||||
## Server headers
|
||||
|
||||
The 402 status code and `X-{payment_method}` header is used by the server to inform the client that a payment is required for the requested operation. The server MUST provide specific headers for each supported payment method.
|
||||
|
||||
Supported payment methods:
|
||||
|
||||
- `X-Cashu`: Payment details for the cashu payment method, adhering to the [NUT-24](https://github.com/cashubtc/nuts/blob/main/24.md) standard.
|
||||
- `X-Lightning`: Payment details for the lightning payment method, adhering to the [BOLT-11](https://github.com/lightning/bolts/blob/master/11-payment-encoding.md) standard.
|
||||
|
||||
If a server supports multiple payment methods, it MAY send multiple `X-{payment_method}` headers in the same response.
|
||||
|
||||
Schema:
|
||||
|
||||
```http
|
||||
HTTP/1.1 402 Payment Required
|
||||
X-{payment_method}: "<encoded_payload_according_to_{payment_method}_spec>"
|
||||
```
|
||||
|
||||
### `X-Cashu` Header
|
||||
|
||||
When using the X-Cashu header, the server MUST adhere to the [NUT-24](https://github.com/cashubtc/nuts/blob/main/24.md) standard.
|
||||
|
||||
Example for cashu:
|
||||
|
||||
```http
|
||||
HTTP/1.1 402 Payment Required
|
||||
X-Cashu: creqApWF0gaNhdGVub3N0cmFheKlucHJvZmlsZTFxeTI4d3VtbjhnaGo3dW45ZDNzaGp0bnl2OWtoMnVld2Q5aHN6OW1od2RlbjV0ZTB3ZmprY2N0ZTljdXJ4dmVuOWVlaHFjdHJ2NWhzenJ0aHdkZW41dGUwZGVoaHh0bnZkYWtxcWd5ZGFxeTdjdXJrNDM5eWtwdGt5c3Y3dWRoZGh1NjhzdWNtMjk1YWtxZWZkZWhrZjBkNDk1Y3d1bmw1YWeBgmFuYjE3YWloYjdhOTAxNzZhYQphdWNzYXRhbYF4Imh0dHBzOi8vbm9mZWVzLnRlc3RudXQuY2FzaHUuc3BhY2U
|
||||
```
|
||||
|
||||
### `X-Lightning` Header
|
||||
|
||||
When using the X-Lightning header, the server MUST adhere to the [BOLT-11](https://github.com/lightning/bolts/blob/master/11-payment-encoding.md) standard.
|
||||
Example for lightning:
|
||||
|
||||
```http
|
||||
HTTP/1.1 402 Payment Required
|
||||
X-Lightning: lnbc30n1pnnmw3lpp57727jjq8zxctahfavqacymellq56l70f7lwfkmhxfjva6dgul2zqhp5w48l28v60yvythn6qvnpq0lez54422a042yaw4kq8arvd68a6n7qcqzzsxqyz5vqsp5sqezejdfaxx5hge83tf59a50h6gagwah59fjn9mw2d5mn278jkys9qxpqysgqt2q2lhjl9kgfaqz864mhlsspftzdyr642lf3zdt6ljqj6wmathdhtgcn0e6f4ym34jl0qkt6gwnllygvzkhdlpq64c6yv3rta2hyzlqp8k28pz
|
||||
```
|
||||
|
||||
### Client implementation
|
||||
|
||||
Clients MUST parse and validate the `X-{payment_method}` header received from the server. The client SHOULD provide a way for the user to complete the payment and retry the request using the same `X-{payment_method}` header.
|
||||
|
||||
The client MUST provide the payment proof when re-trying the request using the same `X-{payment_method}` header that was chosen. The payment proof MUST align with the payment method specification:
|
||||
|
||||
- For cashu the payment proof should be a serialized `cashuB` token in the `X-Cashu` header according to [NUT-24](https://github.com/cashubtc/nuts/blob/main/24.md#client-payment).
|
||||
- For lightning the payment proof should be the preimage of the payment request according to [BOLT-11](https://github.com/lightning/bolts/blob/master/11-payment-encoding.md).
|
||||
|
||||
Schema:
|
||||
|
||||
```http
|
||||
X-{payment_method}: "<encoded_payment_proof_according_to_{payment_method}_spec>"
|
||||
```
|
||||
|
||||
Example for Cashu:
|
||||
|
||||
```http
|
||||
X-Cashu: cashuBo2F0gqJhaUgA_9SLj17PgGFwgaNhYQFhc3hAYWNjMTI0MzVlN2I4NDg0YzNjZjE4NTAxNDkyMThhZjkwZjcxNmE1MmJmNGE1ZWQzNDdlNDhlY2MxM2Y3NzM4OGFjWCECRFODGd5IXVW
|
||||
```
|
||||
|
||||
Example for Lightning:
|
||||
|
||||
```http
|
||||
X-Lightning: 966fcb8f153339372f9a187f725384ff4ceae0047c25b9ce607488d7c7e93bba
|
||||
```
|
||||
|
||||
**Special Note on HEAD Requests**
|
||||
|
||||
The HEAD endpoints are only used to retrieve blob or server information. They MUST NOT be retried with payment proof. Instead, clients should complete the payment and proceed with the `PUT` or `GET` request.
|
||||
|
||||
### Error handling
|
||||
|
||||
If the client fails to provide the payment proof (expired invoice, invalid token, etc.) the server MUST respond with **400 Bad request** status code and include a `X-Reason` header with a human-readable message. The client SHOULD inform the user about the error and provide a way to retry the request.
|
||||
|
||||
### Extending with Future Payment Methods
|
||||
|
||||
To support future payment methods (e.g., other Layer 2 solutions), the specification allows the addition of new X-{payment_method} headers. Each new method MUST adhere to the following:
|
||||
|
||||
New methods MUST use a unique `X-{payment_method}` header containing the specific payment details.
|
||||
|
||||
New methods MUST adhere their own specification, which MUST be publicly available and linked in the header.
|
||||
35
pkg/protocol/blossom/blossom/buds/08.md
Normal file
35
pkg/protocol/blossom/blossom/buds/08.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# BUD-08
|
||||
|
||||
## Nostr File Metadata Tags
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
Describes how a server could return nostr [NIP-94 File Metadata](https://github.com/nostr-protocol/nips/blob/master/94.md) tags from the `/upload` and `/mirror` endpoints
|
||||
|
||||
### Returning tags
|
||||
|
||||
As described in [BUD-02](./02.md#blob-descriptor) servers MAY add any additional fields to a blob descriptor
|
||||
|
||||
Servers MAY return an additional `nip94` field in the [blob descriptor](./02.md#blob-descriptor) from the `/upload` or `/mirror` endpoints
|
||||
|
||||
The `nip94` field should contain a JSON array with KV pairs as defined in [NIP-94](https://github.com/nostr-protocol/nips/blob/master/94.md)
|
||||
|
||||
An example response would look like:
|
||||
|
||||
```json
|
||||
{
|
||||
"url": "https://cdn.example.com/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf",
|
||||
"sha256": "b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553",
|
||||
"size": 184292,
|
||||
"type": "application/pdf",
|
||||
"uploaded": 1725909682,
|
||||
"nip94": [
|
||||
["url", "https://cdn.example.com/b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf"],
|
||||
["m", "application/pdf"],
|
||||
["x", "b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553"],
|
||||
["size", "184292"],
|
||||
["magnet", "magnet:?xt=urn:btih:9804c5286a3fb07b2244c968b39bc3cc814313bc&dn=bitcoin.pdf"],
|
||||
["i", "9804c5286a3fb07b2244c968b39bc3cc814313bc"]
|
||||
]
|
||||
}
|
||||
```
|
||||
40
pkg/protocol/blossom/blossom/buds/09.md
Normal file
40
pkg/protocol/blossom/blossom/buds/09.md
Normal file
@@ -0,0 +1,40 @@
|
||||
# BUD-09
|
||||
|
||||
## Blob Report
|
||||
|
||||
`draft` `optional`
|
||||
|
||||
This bud defines a new endpoint for clients and users to report blobs to servers.
|
||||
|
||||
### PUT /report - reporting a blob
|
||||
|
||||
The request body MUST be a signed [NIP-56](https://github.com/nostr-protocol/nips/blob/master/56.md) report event with one or more `x` tags containing the hashes of the blobs being reported.
|
||||
|
||||
Example:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"kind": 1984,
|
||||
"tags": [
|
||||
["x", "<blob-sha256>", "<type-based-on-nip-56>"],
|
||||
["x", "<another-blob-sha256>", "<type-based-on-nip-56>"]
|
||||
],
|
||||
"content": "<human readable report details>",
|
||||
// other fields...
|
||||
}
|
||||
```
|
||||
|
||||
The clients can include `e` or `p` tags to point to the event or the profile that contains this media if they want to make this report event useful for relays as well.
|
||||
|
||||
Server MUST respond to a report request with a success code or a code in the 4xx/5xx range if there was any error.
|
||||
|
||||
### Client behavior
|
||||
|
||||
The clients can show a blob report button on posts or in blob details. Or its RECOMMENDED to merge this with normal nostr report and send it to both relays and blossom server. other clients can receive it from relays and hide or blur reported blob from trusted friends.
|
||||
|
||||
### Server behavior
|
||||
|
||||
The servers MAY keep the reports somewhere for operators to check and take action on them. they MAY use a list of trusted people or moderators to directly take action on blob without operator request.
|
||||
|
||||
Servers MAY consider removed blobs sha256 as blocked to prevent rewrite.
|
||||
Servers SHOULD advertise a route or landing page to provide their rules and terms of service which affects the report process.
|
||||
237
pkg/ratelimit/badger_monitor.go
Normal file
237
pkg/ratelimit/badger_monitor.go
Normal file
@@ -0,0 +1,237 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"next.orly.dev/pkg/interfaces/loadmonitor"
|
||||
)
|
||||
|
||||
// BadgerMonitor implements loadmonitor.Monitor for the Badger database.
|
||||
// It collects metrics from Badger's LSM tree, caches, and Go runtime.
|
||||
type BadgerMonitor struct {
|
||||
db *badger.DB
|
||||
|
||||
// Target memory for pressure calculation
|
||||
targetMemoryBytes atomic.Uint64
|
||||
|
||||
// Latency tracking with exponential moving average
|
||||
queryLatencyNs atomic.Int64
|
||||
writeLatencyNs atomic.Int64
|
||||
latencyAlpha float64 // EMA coefficient (default 0.1)
|
||||
|
||||
// Cached metrics (updated by background goroutine)
|
||||
metricsLock sync.RWMutex
|
||||
cachedMetrics loadmonitor.Metrics
|
||||
lastL0Tables int
|
||||
lastL0Score float64
|
||||
|
||||
// Background collection
|
||||
stopChan chan struct{}
|
||||
stopped chan struct{}
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// Compile-time check that BadgerMonitor implements loadmonitor.Monitor
|
||||
var _ loadmonitor.Monitor = (*BadgerMonitor)(nil)
|
||||
|
||||
// NewBadgerMonitor creates a new Badger load monitor.
|
||||
// The updateInterval controls how often metrics are collected (default 100ms).
|
||||
func NewBadgerMonitor(db *badger.DB, updateInterval time.Duration) *BadgerMonitor {
|
||||
if updateInterval <= 0 {
|
||||
updateInterval = 100 * time.Millisecond
|
||||
}
|
||||
|
||||
m := &BadgerMonitor{
|
||||
db: db,
|
||||
latencyAlpha: 0.1, // 10% new, 90% old for smooth EMA
|
||||
stopChan: make(chan struct{}),
|
||||
stopped: make(chan struct{}),
|
||||
interval: updateInterval,
|
||||
}
|
||||
|
||||
// Set a default target (1.5GB)
|
||||
m.targetMemoryBytes.Store(1500 * 1024 * 1024)
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// GetMetrics returns the current load metrics.
|
||||
func (m *BadgerMonitor) GetMetrics() loadmonitor.Metrics {
|
||||
m.metricsLock.RLock()
|
||||
defer m.metricsLock.RUnlock()
|
||||
return m.cachedMetrics
|
||||
}
|
||||
|
||||
// RecordQueryLatency records a query latency sample using exponential moving average.
|
||||
func (m *BadgerMonitor) RecordQueryLatency(latency time.Duration) {
|
||||
ns := latency.Nanoseconds()
|
||||
for {
|
||||
old := m.queryLatencyNs.Load()
|
||||
if old == 0 {
|
||||
if m.queryLatencyNs.CompareAndSwap(0, ns) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
// EMA: new = alpha * sample + (1-alpha) * old
|
||||
newVal := int64(m.latencyAlpha*float64(ns) + (1-m.latencyAlpha)*float64(old))
|
||||
if m.queryLatencyNs.CompareAndSwap(old, newVal) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RecordWriteLatency records a write latency sample using exponential moving average.
|
||||
func (m *BadgerMonitor) RecordWriteLatency(latency time.Duration) {
|
||||
ns := latency.Nanoseconds()
|
||||
for {
|
||||
old := m.writeLatencyNs.Load()
|
||||
if old == 0 {
|
||||
if m.writeLatencyNs.CompareAndSwap(0, ns) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
// EMA: new = alpha * sample + (1-alpha) * old
|
||||
newVal := int64(m.latencyAlpha*float64(ns) + (1-m.latencyAlpha)*float64(old))
|
||||
if m.writeLatencyNs.CompareAndSwap(old, newVal) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetMemoryTarget sets the target memory limit in bytes.
|
||||
func (m *BadgerMonitor) SetMemoryTarget(bytes uint64) {
|
||||
m.targetMemoryBytes.Store(bytes)
|
||||
}
|
||||
|
||||
// Start begins background metric collection.
|
||||
func (m *BadgerMonitor) Start() <-chan struct{} {
|
||||
go m.collectLoop()
|
||||
return m.stopped
|
||||
}
|
||||
|
||||
// Stop halts background metric collection.
|
||||
func (m *BadgerMonitor) Stop() {
|
||||
close(m.stopChan)
|
||||
<-m.stopped
|
||||
}
|
||||
|
||||
// collectLoop periodically collects metrics from Badger.
|
||||
func (m *BadgerMonitor) collectLoop() {
|
||||
defer close(m.stopped)
|
||||
|
||||
ticker := time.NewTicker(m.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-m.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.updateMetrics()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateMetrics collects current metrics from Badger and runtime.
|
||||
func (m *BadgerMonitor) updateMetrics() {
|
||||
if m.db == nil || m.db.IsClosed() {
|
||||
return
|
||||
}
|
||||
|
||||
metrics := loadmonitor.Metrics{
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
|
||||
// Calculate memory pressure from Go runtime
|
||||
var memStats runtime.MemStats
|
||||
runtime.ReadMemStats(&memStats)
|
||||
|
||||
targetBytes := m.targetMemoryBytes.Load()
|
||||
if targetBytes > 0 {
|
||||
// Use HeapAlloc as primary memory metric
|
||||
// This represents the actual live heap objects
|
||||
metrics.MemoryPressure = float64(memStats.HeapAlloc) / float64(targetBytes)
|
||||
}
|
||||
|
||||
// Get Badger LSM tree information for write load
|
||||
levels := m.db.Levels()
|
||||
var l0Tables int
|
||||
var maxScore float64
|
||||
|
||||
for _, level := range levels {
|
||||
if level.Level == 0 {
|
||||
l0Tables = level.NumTables
|
||||
}
|
||||
if level.Score > maxScore {
|
||||
maxScore = level.Score
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate write load based on L0 tables and compaction score
|
||||
// L0 tables stall at NumLevelZeroTablesStall (default 16)
|
||||
// We consider write pressure high when approaching that limit
|
||||
const l0StallThreshold = 16
|
||||
l0Load := float64(l0Tables) / float64(l0StallThreshold)
|
||||
if l0Load > 1.0 {
|
||||
l0Load = 1.0
|
||||
}
|
||||
|
||||
// Compaction score > 1.0 means compaction is needed
|
||||
// We blend L0 tables and compaction score for write load
|
||||
compactionLoad := maxScore / 2.0 // Score of 2.0 = fully loaded
|
||||
if compactionLoad > 1.0 {
|
||||
compactionLoad = 1.0
|
||||
}
|
||||
|
||||
// Blend: 60% L0 (immediate backpressure), 40% compaction score
|
||||
metrics.WriteLoad = 0.6*l0Load + 0.4*compactionLoad
|
||||
|
||||
// Calculate read load from cache metrics
|
||||
blockMetrics := m.db.BlockCacheMetrics()
|
||||
indexMetrics := m.db.IndexCacheMetrics()
|
||||
|
||||
var blockHitRatio, indexHitRatio float64
|
||||
if blockMetrics != nil {
|
||||
blockHitRatio = blockMetrics.Ratio()
|
||||
}
|
||||
if indexMetrics != nil {
|
||||
indexHitRatio = indexMetrics.Ratio()
|
||||
}
|
||||
|
||||
// Average cache hit ratio (0 = no hits = high load, 1 = all hits = low load)
|
||||
avgHitRatio := (blockHitRatio + indexHitRatio) / 2.0
|
||||
|
||||
// Invert: low hit ratio = high read load
|
||||
// Use 0.5 as the threshold (below 50% hit ratio is concerning)
|
||||
if avgHitRatio < 0.5 {
|
||||
metrics.ReadLoad = 1.0 - avgHitRatio*2 // 0% hits = 1.0 load, 50% hits = 0.0 load
|
||||
} else {
|
||||
metrics.ReadLoad = 0 // Above 50% hit ratio = minimal load
|
||||
}
|
||||
|
||||
// Store latencies
|
||||
metrics.QueryLatency = time.Duration(m.queryLatencyNs.Load())
|
||||
metrics.WriteLatency = time.Duration(m.writeLatencyNs.Load())
|
||||
|
||||
// Update cached metrics
|
||||
m.metricsLock.Lock()
|
||||
m.cachedMetrics = metrics
|
||||
m.lastL0Tables = l0Tables
|
||||
m.lastL0Score = maxScore
|
||||
m.metricsLock.Unlock()
|
||||
}
|
||||
|
||||
// GetL0Stats returns L0-specific statistics for debugging.
|
||||
func (m *BadgerMonitor) GetL0Stats() (tables int, score float64) {
|
||||
m.metricsLock.RLock()
|
||||
defer m.metricsLock.RUnlock()
|
||||
return m.lastL0Tables, m.lastL0Score
|
||||
}
|
||||
56
pkg/ratelimit/factory.go
Normal file
56
pkg/ratelimit/factory.go
Normal file
@@ -0,0 +1,56 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"next.orly.dev/pkg/interfaces/loadmonitor"
|
||||
)
|
||||
|
||||
// NewBadgerLimiter creates a rate limiter configured for a Badger database.
|
||||
// It automatically creates a BadgerMonitor for the provided database.
|
||||
func NewBadgerLimiter(config Config, db *badger.DB) *Limiter {
|
||||
monitor := NewBadgerMonitor(db, 100*time.Millisecond)
|
||||
return NewLimiter(config, monitor)
|
||||
}
|
||||
|
||||
// NewNeo4jLimiter creates a rate limiter configured for a Neo4j database.
|
||||
// It automatically creates a Neo4jMonitor for the provided driver.
|
||||
// querySem should be the semaphore used to limit concurrent queries.
|
||||
// maxConcurrency is typically 10 (matching the semaphore size).
|
||||
func NewNeo4jLimiter(
|
||||
config Config,
|
||||
driver neo4j.DriverWithContext,
|
||||
querySem chan struct{},
|
||||
maxConcurrency int,
|
||||
) *Limiter {
|
||||
monitor := NewNeo4jMonitor(driver, querySem, maxConcurrency, 100*time.Millisecond)
|
||||
return NewLimiter(config, monitor)
|
||||
}
|
||||
|
||||
// NewDisabledLimiter creates a rate limiter that is disabled.
|
||||
// This is useful when rate limiting is not configured.
|
||||
func NewDisabledLimiter() *Limiter {
|
||||
config := DefaultConfig()
|
||||
config.Enabled = false
|
||||
return NewLimiter(config, nil)
|
||||
}
|
||||
|
||||
// MonitorFromBadgerDB creates a BadgerMonitor from a Badger database.
|
||||
// Exported for use when you need to create the monitor separately.
|
||||
func MonitorFromBadgerDB(db *badger.DB) loadmonitor.Monitor {
|
||||
return NewBadgerMonitor(db, 100*time.Millisecond)
|
||||
}
|
||||
|
||||
// MonitorFromNeo4jDriver creates a Neo4jMonitor from a Neo4j driver.
|
||||
// Exported for use when you need to create the monitor separately.
|
||||
func MonitorFromNeo4jDriver(
|
||||
driver neo4j.DriverWithContext,
|
||||
querySem chan struct{},
|
||||
maxConcurrency int,
|
||||
) loadmonitor.Monitor {
|
||||
return NewNeo4jMonitor(driver, querySem, maxConcurrency, 100*time.Millisecond)
|
||||
}
|
||||
434
pkg/ratelimit/limiter.go
Normal file
434
pkg/ratelimit/limiter.go
Normal file
@@ -0,0 +1,434 @@
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"next.orly.dev/pkg/interfaces/loadmonitor"
|
||||
pidif "next.orly.dev/pkg/interfaces/pid"
|
||||
"next.orly.dev/pkg/pid"
|
||||
)
|
||||
|
||||
// OperationType distinguishes between read and write operations
|
||||
// for applying different rate limiting strategies.
|
||||
type OperationType int
|
||||
|
||||
const (
|
||||
// Read operations (REQ queries)
|
||||
Read OperationType = iota
|
||||
// Write operations (EVENT saves, imports)
|
||||
Write
|
||||
)
|
||||
|
||||
// String returns a human-readable name for the operation type.
|
||||
func (o OperationType) String() string {
|
||||
switch o {
|
||||
case Read:
|
||||
return "read"
|
||||
case Write:
|
||||
return "write"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// Config holds configuration for the adaptive rate limiter.
|
||||
type Config struct {
|
||||
// Enabled controls whether rate limiting is active.
|
||||
Enabled bool
|
||||
|
||||
// TargetMemoryMB is the target memory limit in megabytes.
|
||||
// Memory pressure is calculated relative to this target.
|
||||
TargetMemoryMB int
|
||||
|
||||
// WriteSetpoint is the target process variable for writes (0.0-1.0).
|
||||
// Default: 0.85 (throttle when load exceeds 85%)
|
||||
WriteSetpoint float64
|
||||
|
||||
// ReadSetpoint is the target process variable for reads (0.0-1.0).
|
||||
// Default: 0.90 (more tolerant for reads)
|
||||
ReadSetpoint float64
|
||||
|
||||
// PID gains for writes
|
||||
WriteKp float64
|
||||
WriteKi float64
|
||||
WriteKd float64
|
||||
|
||||
// PID gains for reads
|
||||
ReadKp float64
|
||||
ReadKi float64
|
||||
ReadKd float64
|
||||
|
||||
// MaxWriteDelayMs is the maximum delay for write operations in milliseconds.
|
||||
MaxWriteDelayMs int
|
||||
|
||||
// MaxReadDelayMs is the maximum delay for read operations in milliseconds.
|
||||
MaxReadDelayMs int
|
||||
|
||||
// MetricUpdateInterval is how often to poll the load monitor.
|
||||
MetricUpdateInterval time.Duration
|
||||
|
||||
// MemoryWeight is the weight given to memory pressure in process variable (0.0-1.0).
|
||||
// The remaining weight is given to the load metric.
|
||||
// Default: 0.7 (70% memory, 30% load)
|
||||
MemoryWeight float64
|
||||
}
|
||||
|
||||
// DefaultConfig returns a default configuration for the rate limiter.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Enabled: true,
|
||||
TargetMemoryMB: 1500, // 1.5GB target
|
||||
WriteSetpoint: 0.85,
|
||||
ReadSetpoint: 0.90,
|
||||
WriteKp: 0.5,
|
||||
WriteKi: 0.1,
|
||||
WriteKd: 0.05,
|
||||
ReadKp: 0.3,
|
||||
ReadKi: 0.05,
|
||||
ReadKd: 0.02,
|
||||
MaxWriteDelayMs: 1000, // 1 second max
|
||||
MaxReadDelayMs: 500, // 500ms max
|
||||
MetricUpdateInterval: 100 * time.Millisecond,
|
||||
MemoryWeight: 0.7,
|
||||
}
|
||||
}
|
||||
|
||||
// NewConfigFromValues creates a Config from individual configuration values.
|
||||
// This is useful when loading configuration from environment variables.
|
||||
func NewConfigFromValues(
|
||||
enabled bool,
|
||||
targetMB int,
|
||||
writeKp, writeKi, writeKd float64,
|
||||
readKp, readKi, readKd float64,
|
||||
maxWriteMs, maxReadMs int,
|
||||
writeTarget, readTarget float64,
|
||||
) Config {
|
||||
return Config{
|
||||
Enabled: enabled,
|
||||
TargetMemoryMB: targetMB,
|
||||
WriteSetpoint: writeTarget,
|
||||
ReadSetpoint: readTarget,
|
||||
WriteKp: writeKp,
|
||||
WriteKi: writeKi,
|
||||
WriteKd: writeKd,
|
||||
ReadKp: readKp,
|
||||
ReadKi: readKi,
|
||||
ReadKd: readKd,
|
||||
MaxWriteDelayMs: maxWriteMs,
|
||||
MaxReadDelayMs: maxReadMs,
|
||||
MetricUpdateInterval: 100 * time.Millisecond,
|
||||
MemoryWeight: 0.7,
|
||||
}
|
||||
}
|
||||
|
||||
// Limiter implements adaptive rate limiting using PID control.
|
||||
// It monitors database load metrics and computes appropriate delays
|
||||
// to keep the system within its target operating range.
|
||||
type Limiter struct {
|
||||
config Config
|
||||
monitor loadmonitor.Monitor
|
||||
|
||||
// PID controllers for reads and writes (using generic pid.Controller)
|
||||
writePID pidif.Controller
|
||||
readPID pidif.Controller
|
||||
|
||||
// Cached metrics (updated periodically)
|
||||
metricsLock sync.RWMutex
|
||||
currentMetrics loadmonitor.Metrics
|
||||
|
||||
// Statistics
|
||||
totalWriteDelayMs atomic.Int64
|
||||
totalReadDelayMs atomic.Int64
|
||||
writeThrottles atomic.Int64
|
||||
readThrottles atomic.Int64
|
||||
|
||||
// Lifecycle
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
stopOnce sync.Once
|
||||
stopped chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewLimiter creates a new adaptive rate limiter.
|
||||
// If monitor is nil, the limiter will be disabled.
|
||||
func NewLimiter(config Config, monitor loadmonitor.Monitor) *Limiter {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
l := &Limiter{
|
||||
config: config,
|
||||
monitor: monitor,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
stopped: make(chan struct{}),
|
||||
}
|
||||
|
||||
// Create PID controllers with configured gains using the generic pid package
|
||||
l.writePID = pid.New(pidif.Tuning{
|
||||
Kp: config.WriteKp,
|
||||
Ki: config.WriteKi,
|
||||
Kd: config.WriteKd,
|
||||
Setpoint: config.WriteSetpoint,
|
||||
DerivativeFilterAlpha: 0.2, // Strong filtering for writes
|
||||
IntegralMin: -2.0,
|
||||
IntegralMax: float64(config.MaxWriteDelayMs) / 1000.0 * 2, // Anti-windup limits
|
||||
OutputMin: 0,
|
||||
OutputMax: float64(config.MaxWriteDelayMs) / 1000.0,
|
||||
})
|
||||
|
||||
l.readPID = pid.New(pidif.Tuning{
|
||||
Kp: config.ReadKp,
|
||||
Ki: config.ReadKi,
|
||||
Kd: config.ReadKd,
|
||||
Setpoint: config.ReadSetpoint,
|
||||
DerivativeFilterAlpha: 0.15, // Very strong filtering for reads
|
||||
IntegralMin: -1.0,
|
||||
IntegralMax: float64(config.MaxReadDelayMs) / 1000.0 * 2,
|
||||
OutputMin: 0,
|
||||
OutputMax: float64(config.MaxReadDelayMs) / 1000.0,
|
||||
})
|
||||
|
||||
// Set memory target on monitor
|
||||
if monitor != nil && config.TargetMemoryMB > 0 {
|
||||
monitor.SetMemoryTarget(uint64(config.TargetMemoryMB) * 1024 * 1024)
|
||||
}
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
// Start begins the rate limiter's background metric collection.
|
||||
func (l *Limiter) Start() {
|
||||
if l.monitor == nil || !l.config.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
// Start the monitor
|
||||
l.monitor.Start()
|
||||
|
||||
// Start metric update loop
|
||||
l.wg.Add(1)
|
||||
go l.updateLoop()
|
||||
}
|
||||
|
||||
// updateLoop periodically fetches metrics from the monitor.
|
||||
func (l *Limiter) updateLoop() {
|
||||
defer l.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(l.config.MetricUpdateInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-l.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if l.monitor != nil {
|
||||
metrics := l.monitor.GetMetrics()
|
||||
l.metricsLock.Lock()
|
||||
l.currentMetrics = metrics
|
||||
l.metricsLock.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop halts the rate limiter.
|
||||
func (l *Limiter) Stop() {
|
||||
l.stopOnce.Do(func() {
|
||||
l.cancel()
|
||||
if l.monitor != nil {
|
||||
l.monitor.Stop()
|
||||
}
|
||||
l.wg.Wait()
|
||||
close(l.stopped)
|
||||
})
|
||||
}
|
||||
|
||||
// Stopped returns a channel that closes when the limiter has stopped.
|
||||
func (l *Limiter) Stopped() <-chan struct{} {
|
||||
return l.stopped
|
||||
}
|
||||
|
||||
// Wait blocks until the rate limiter permits the operation to proceed.
|
||||
// It returns the delay that was applied, or 0 if no delay was needed.
|
||||
// If the context is cancelled, it returns immediately.
|
||||
func (l *Limiter) Wait(ctx context.Context, opType OperationType) time.Duration {
|
||||
if !l.config.Enabled || l.monitor == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
delay := l.ComputeDelay(opType)
|
||||
if delay <= 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Apply the delay
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return 0
|
||||
case <-time.After(delay):
|
||||
return delay
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeDelay calculates the recommended delay for an operation.
|
||||
// This can be used to check the delay without actually waiting.
|
||||
func (l *Limiter) ComputeDelay(opType OperationType) time.Duration {
|
||||
if !l.config.Enabled || l.monitor == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Get current metrics
|
||||
l.metricsLock.RLock()
|
||||
metrics := l.currentMetrics
|
||||
l.metricsLock.RUnlock()
|
||||
|
||||
// Compute process variable as weighted combination of memory and load
|
||||
var loadMetric float64
|
||||
switch opType {
|
||||
case Write:
|
||||
loadMetric = metrics.WriteLoad
|
||||
case Read:
|
||||
loadMetric = metrics.ReadLoad
|
||||
}
|
||||
|
||||
// Combine memory pressure and load
|
||||
// Process variable = memoryWeight * memoryPressure + (1-memoryWeight) * loadMetric
|
||||
pv := l.config.MemoryWeight*metrics.MemoryPressure + (1-l.config.MemoryWeight)*loadMetric
|
||||
|
||||
// Select the appropriate PID controller
|
||||
var delaySec float64
|
||||
switch opType {
|
||||
case Write:
|
||||
out := l.writePID.UpdateValue(pv)
|
||||
delaySec = out.Value()
|
||||
if delaySec > 0 {
|
||||
l.writeThrottles.Add(1)
|
||||
l.totalWriteDelayMs.Add(int64(delaySec * 1000))
|
||||
}
|
||||
case Read:
|
||||
out := l.readPID.UpdateValue(pv)
|
||||
delaySec = out.Value()
|
||||
if delaySec > 0 {
|
||||
l.readThrottles.Add(1)
|
||||
l.totalReadDelayMs.Add(int64(delaySec * 1000))
|
||||
}
|
||||
}
|
||||
|
||||
if delaySec <= 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return time.Duration(delaySec * float64(time.Second))
|
||||
}
|
||||
|
||||
// RecordLatency records an operation latency for the monitor.
|
||||
func (l *Limiter) RecordLatency(opType OperationType, latency time.Duration) {
|
||||
if l.monitor == nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch opType {
|
||||
case Write:
|
||||
l.monitor.RecordWriteLatency(latency)
|
||||
case Read:
|
||||
l.monitor.RecordQueryLatency(latency)
|
||||
}
|
||||
}
|
||||
|
||||
// Stats returns rate limiter statistics.
|
||||
type Stats struct {
|
||||
WriteThrottles int64
|
||||
ReadThrottles int64
|
||||
TotalWriteDelayMs int64
|
||||
TotalReadDelayMs int64
|
||||
CurrentMetrics loadmonitor.Metrics
|
||||
WritePIDState PIDState
|
||||
ReadPIDState PIDState
|
||||
}
|
||||
|
||||
// PIDState contains the internal state of a PID controller.
|
||||
type PIDState struct {
|
||||
Integral float64
|
||||
PrevError float64
|
||||
PrevFilteredError float64
|
||||
}
|
||||
|
||||
// GetStats returns current rate limiter statistics.
|
||||
func (l *Limiter) GetStats() Stats {
|
||||
l.metricsLock.RLock()
|
||||
metrics := l.currentMetrics
|
||||
l.metricsLock.RUnlock()
|
||||
|
||||
stats := Stats{
|
||||
WriteThrottles: l.writeThrottles.Load(),
|
||||
ReadThrottles: l.readThrottles.Load(),
|
||||
TotalWriteDelayMs: l.totalWriteDelayMs.Load(),
|
||||
TotalReadDelayMs: l.totalReadDelayMs.Load(),
|
||||
CurrentMetrics: metrics,
|
||||
}
|
||||
|
||||
// Type assert to concrete pid.Controller to access State() method
|
||||
// This is for monitoring/debugging only
|
||||
if wCtrl, ok := l.writePID.(*pid.Controller); ok {
|
||||
integral, prevErr, prevFiltered, _ := wCtrl.State()
|
||||
stats.WritePIDState = PIDState{
|
||||
Integral: integral,
|
||||
PrevError: prevErr,
|
||||
PrevFilteredError: prevFiltered,
|
||||
}
|
||||
}
|
||||
if rCtrl, ok := l.readPID.(*pid.Controller); ok {
|
||||
integral, prevErr, prevFiltered, _ := rCtrl.State()
|
||||
stats.ReadPIDState = PIDState{
|
||||
Integral: integral,
|
||||
PrevError: prevErr,
|
||||
PrevFilteredError: prevFiltered,
|
||||
}
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// Reset clears all PID controller state and statistics.
|
||||
func (l *Limiter) Reset() {
|
||||
l.writePID.Reset()
|
||||
l.readPID.Reset()
|
||||
l.writeThrottles.Store(0)
|
||||
l.readThrottles.Store(0)
|
||||
l.totalWriteDelayMs.Store(0)
|
||||
l.totalReadDelayMs.Store(0)
|
||||
}
|
||||
|
||||
// IsEnabled returns whether rate limiting is active.
|
||||
func (l *Limiter) IsEnabled() bool {
|
||||
return l.config.Enabled && l.monitor != nil
|
||||
}
|
||||
|
||||
// UpdateConfig updates the rate limiter configuration.
|
||||
// This is useful for dynamic tuning.
|
||||
func (l *Limiter) UpdateConfig(config Config) {
|
||||
l.config = config
|
||||
|
||||
// Update PID controllers - use interface methods for setpoint and gains
|
||||
l.writePID.SetSetpoint(config.WriteSetpoint)
|
||||
l.writePID.SetGains(config.WriteKp, config.WriteKi, config.WriteKd)
|
||||
// Type assert to set output limits (not part of base interface)
|
||||
if wCtrl, ok := l.writePID.(*pid.Controller); ok {
|
||||
wCtrl.SetOutputLimits(0, float64(config.MaxWriteDelayMs)/1000.0)
|
||||
}
|
||||
|
||||
l.readPID.SetSetpoint(config.ReadSetpoint)
|
||||
l.readPID.SetGains(config.ReadKp, config.ReadKi, config.ReadKd)
|
||||
if rCtrl, ok := l.readPID.(*pid.Controller); ok {
|
||||
rCtrl.SetOutputLimits(0, float64(config.MaxReadDelayMs)/1000.0)
|
||||
}
|
||||
|
||||
// Update memory target
|
||||
if l.monitor != nil && config.TargetMemoryMB > 0 {
|
||||
l.monitor.SetMemoryTarget(uint64(config.TargetMemoryMB) * 1024 * 1024)
|
||||
}
|
||||
}
|
||||
259
pkg/ratelimit/neo4j_monitor.go
Normal file
259
pkg/ratelimit/neo4j_monitor.go
Normal file
@@ -0,0 +1,259 @@
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"next.orly.dev/pkg/interfaces/loadmonitor"
|
||||
)
|
||||
|
||||
// Neo4jMonitor implements loadmonitor.Monitor for Neo4j database.
|
||||
// Since Neo4j driver doesn't expose detailed metrics, we track:
|
||||
// - Memory pressure via Go runtime
|
||||
// - Query concurrency via the semaphore
|
||||
// - Latency via recording
|
||||
type Neo4jMonitor struct {
|
||||
driver neo4j.DriverWithContext
|
||||
querySem chan struct{} // Reference to the query semaphore
|
||||
|
||||
// Target memory for pressure calculation
|
||||
targetMemoryBytes atomic.Uint64
|
||||
|
||||
// Latency tracking with exponential moving average
|
||||
queryLatencyNs atomic.Int64
|
||||
writeLatencyNs atomic.Int64
|
||||
latencyAlpha float64 // EMA coefficient (default 0.1)
|
||||
|
||||
// Concurrency tracking
|
||||
activeReads atomic.Int32
|
||||
activeWrites atomic.Int32
|
||||
maxConcurrency int
|
||||
|
||||
// Cached metrics (updated by background goroutine)
|
||||
metricsLock sync.RWMutex
|
||||
cachedMetrics loadmonitor.Metrics
|
||||
|
||||
// Background collection
|
||||
stopChan chan struct{}
|
||||
stopped chan struct{}
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// Compile-time check that Neo4jMonitor implements loadmonitor.Monitor
|
||||
var _ loadmonitor.Monitor = (*Neo4jMonitor)(nil)
|
||||
|
||||
// NewNeo4jMonitor creates a new Neo4j load monitor.
|
||||
// The querySem should be the same semaphore used for limiting concurrent queries.
|
||||
// maxConcurrency is the maximum concurrent query limit (typically 10).
|
||||
func NewNeo4jMonitor(
|
||||
driver neo4j.DriverWithContext,
|
||||
querySem chan struct{},
|
||||
maxConcurrency int,
|
||||
updateInterval time.Duration,
|
||||
) *Neo4jMonitor {
|
||||
if updateInterval <= 0 {
|
||||
updateInterval = 100 * time.Millisecond
|
||||
}
|
||||
if maxConcurrency <= 0 {
|
||||
maxConcurrency = 10
|
||||
}
|
||||
|
||||
m := &Neo4jMonitor{
|
||||
driver: driver,
|
||||
querySem: querySem,
|
||||
maxConcurrency: maxConcurrency,
|
||||
latencyAlpha: 0.1, // 10% new, 90% old for smooth EMA
|
||||
stopChan: make(chan struct{}),
|
||||
stopped: make(chan struct{}),
|
||||
interval: updateInterval,
|
||||
}
|
||||
|
||||
// Set a default target (1.5GB)
|
||||
m.targetMemoryBytes.Store(1500 * 1024 * 1024)
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// GetMetrics returns the current load metrics.
|
||||
func (m *Neo4jMonitor) GetMetrics() loadmonitor.Metrics {
|
||||
m.metricsLock.RLock()
|
||||
defer m.metricsLock.RUnlock()
|
||||
return m.cachedMetrics
|
||||
}
|
||||
|
||||
// RecordQueryLatency records a query latency sample using exponential moving average.
|
||||
func (m *Neo4jMonitor) RecordQueryLatency(latency time.Duration) {
|
||||
ns := latency.Nanoseconds()
|
||||
for {
|
||||
old := m.queryLatencyNs.Load()
|
||||
if old == 0 {
|
||||
if m.queryLatencyNs.CompareAndSwap(0, ns) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
// EMA: new = alpha * sample + (1-alpha) * old
|
||||
newVal := int64(m.latencyAlpha*float64(ns) + (1-m.latencyAlpha)*float64(old))
|
||||
if m.queryLatencyNs.CompareAndSwap(old, newVal) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RecordWriteLatency records a write latency sample using exponential moving average.
|
||||
func (m *Neo4jMonitor) RecordWriteLatency(latency time.Duration) {
|
||||
ns := latency.Nanoseconds()
|
||||
for {
|
||||
old := m.writeLatencyNs.Load()
|
||||
if old == 0 {
|
||||
if m.writeLatencyNs.CompareAndSwap(0, ns) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
// EMA: new = alpha * sample + (1-alpha) * old
|
||||
newVal := int64(m.latencyAlpha*float64(ns) + (1-m.latencyAlpha)*float64(old))
|
||||
if m.writeLatencyNs.CompareAndSwap(old, newVal) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetMemoryTarget sets the target memory limit in bytes.
|
||||
func (m *Neo4jMonitor) SetMemoryTarget(bytes uint64) {
|
||||
m.targetMemoryBytes.Store(bytes)
|
||||
}
|
||||
|
||||
// Start begins background metric collection.
|
||||
func (m *Neo4jMonitor) Start() <-chan struct{} {
|
||||
go m.collectLoop()
|
||||
return m.stopped
|
||||
}
|
||||
|
||||
// Stop halts background metric collection.
|
||||
func (m *Neo4jMonitor) Stop() {
|
||||
close(m.stopChan)
|
||||
<-m.stopped
|
||||
}
|
||||
|
||||
// collectLoop periodically collects metrics.
|
||||
func (m *Neo4jMonitor) collectLoop() {
|
||||
defer close(m.stopped)
|
||||
|
||||
ticker := time.NewTicker(m.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-m.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.updateMetrics()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateMetrics collects current metrics.
|
||||
func (m *Neo4jMonitor) updateMetrics() {
|
||||
metrics := loadmonitor.Metrics{
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
|
||||
// Calculate memory pressure from Go runtime
|
||||
var memStats runtime.MemStats
|
||||
runtime.ReadMemStats(&memStats)
|
||||
|
||||
targetBytes := m.targetMemoryBytes.Load()
|
||||
if targetBytes > 0 {
|
||||
// Use HeapAlloc as primary memory metric
|
||||
metrics.MemoryPressure = float64(memStats.HeapAlloc) / float64(targetBytes)
|
||||
}
|
||||
|
||||
// Calculate load from semaphore usage
|
||||
// querySem is a buffered channel - count how many slots are taken
|
||||
if m.querySem != nil {
|
||||
usedSlots := len(m.querySem)
|
||||
concurrencyLoad := float64(usedSlots) / float64(m.maxConcurrency)
|
||||
if concurrencyLoad > 1.0 {
|
||||
concurrencyLoad = 1.0
|
||||
}
|
||||
// Both read and write use the same semaphore
|
||||
metrics.WriteLoad = concurrencyLoad
|
||||
metrics.ReadLoad = concurrencyLoad
|
||||
}
|
||||
|
||||
// Add latency-based load adjustment
|
||||
// High latency indicates the database is struggling
|
||||
queryLatencyNs := m.queryLatencyNs.Load()
|
||||
writeLatencyNs := m.writeLatencyNs.Load()
|
||||
|
||||
// Consider > 500ms query latency as concerning
|
||||
const latencyThresholdNs = 500 * 1e6 // 500ms
|
||||
if queryLatencyNs > 0 {
|
||||
latencyLoad := float64(queryLatencyNs) / float64(latencyThresholdNs)
|
||||
if latencyLoad > 1.0 {
|
||||
latencyLoad = 1.0
|
||||
}
|
||||
// Blend concurrency and latency for read load
|
||||
metrics.ReadLoad = 0.5*metrics.ReadLoad + 0.5*latencyLoad
|
||||
}
|
||||
|
||||
if writeLatencyNs > 0 {
|
||||
latencyLoad := float64(writeLatencyNs) / float64(latencyThresholdNs)
|
||||
if latencyLoad > 1.0 {
|
||||
latencyLoad = 1.0
|
||||
}
|
||||
// Blend concurrency and latency for write load
|
||||
metrics.WriteLoad = 0.5*metrics.WriteLoad + 0.5*latencyLoad
|
||||
}
|
||||
|
||||
// Store latencies
|
||||
metrics.QueryLatency = time.Duration(queryLatencyNs)
|
||||
metrics.WriteLatency = time.Duration(writeLatencyNs)
|
||||
|
||||
// Update cached metrics
|
||||
m.metricsLock.Lock()
|
||||
m.cachedMetrics = metrics
|
||||
m.metricsLock.Unlock()
|
||||
}
|
||||
|
||||
// IncrementActiveReads tracks an active read operation.
|
||||
// Call this when starting a read, and call the returned function when done.
|
||||
func (m *Neo4jMonitor) IncrementActiveReads() func() {
|
||||
m.activeReads.Add(1)
|
||||
return func() {
|
||||
m.activeReads.Add(-1)
|
||||
}
|
||||
}
|
||||
|
||||
// IncrementActiveWrites tracks an active write operation.
|
||||
// Call this when starting a write, and call the returned function when done.
|
||||
func (m *Neo4jMonitor) IncrementActiveWrites() func() {
|
||||
m.activeWrites.Add(1)
|
||||
return func() {
|
||||
m.activeWrites.Add(-1)
|
||||
}
|
||||
}
|
||||
|
||||
// GetConcurrencyStats returns current concurrency statistics for debugging.
|
||||
func (m *Neo4jMonitor) GetConcurrencyStats() (reads, writes int32, semUsed int) {
|
||||
reads = m.activeReads.Load()
|
||||
writes = m.activeWrites.Load()
|
||||
if m.querySem != nil {
|
||||
semUsed = len(m.querySem)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// CheckConnectivity performs a connectivity check to Neo4j.
|
||||
// This can be used to verify the database is responsive.
|
||||
func (m *Neo4jMonitor) CheckConnectivity(ctx context.Context) error {
|
||||
if m.driver == nil {
|
||||
return nil
|
||||
}
|
||||
return m.driver.VerifyConnectivity(ctx)
|
||||
}
|
||||
218
pkg/ratelimit/pid.go
Normal file
218
pkg/ratelimit/pid.go
Normal file
@@ -0,0 +1,218 @@
|
||||
// Package ratelimit provides adaptive rate limiting using PID control.
|
||||
// The PID controller uses proportional, integral, and derivative terms
|
||||
// with a low-pass filter on the derivative to suppress high-frequency noise.
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PIDController implements a PID controller with filtered derivative.
|
||||
// It is designed for rate limiting database operations based on load metrics.
|
||||
//
|
||||
// The controller computes a delay recommendation based on:
|
||||
// - Proportional (P): Immediate response to current error
|
||||
// - Integral (I): Accumulated error to eliminate steady-state offset
|
||||
// - Derivative (D): Rate of change prediction (filtered to reduce noise)
|
||||
//
|
||||
// The filtered derivative uses a low-pass filter to attenuate high-frequency
|
||||
// noise that would otherwise cause erratic control behavior.
|
||||
type PIDController struct {
|
||||
// Gains
|
||||
Kp float64 // Proportional gain
|
||||
Ki float64 // Integral gain
|
||||
Kd float64 // Derivative gain
|
||||
|
||||
// Setpoint is the target process variable value (e.g., 0.85 for 85% of target memory).
|
||||
// The controller drives the process variable toward this setpoint.
|
||||
Setpoint float64
|
||||
|
||||
// DerivativeFilterAlpha is the low-pass filter coefficient for the derivative term.
|
||||
// Range: 0.0-1.0, where lower values provide stronger filtering.
|
||||
// Recommended: 0.2 for strong filtering, 0.5 for moderate filtering.
|
||||
DerivativeFilterAlpha float64
|
||||
|
||||
// Integral limits for anti-windup
|
||||
IntegralMax float64
|
||||
IntegralMin float64
|
||||
|
||||
// Output limits
|
||||
OutputMin float64 // Minimum output (typically 0 = no delay)
|
||||
OutputMax float64 // Maximum output (max delay in seconds)
|
||||
|
||||
// Internal state (protected by mutex)
|
||||
mu sync.Mutex
|
||||
integral float64
|
||||
prevError float64
|
||||
prevFilteredError float64
|
||||
lastUpdate time.Time
|
||||
initialized bool
|
||||
}
|
||||
|
||||
// DefaultPIDControllerForWrites creates a PID controller tuned for write operations.
|
||||
// Writes benefit from aggressive integral and moderate proportional response.
|
||||
func DefaultPIDControllerForWrites() *PIDController {
|
||||
return &PIDController{
|
||||
Kp: 0.5, // Moderate proportional response
|
||||
Ki: 0.1, // Steady integral to eliminate offset
|
||||
Kd: 0.05, // Small derivative for prediction
|
||||
Setpoint: 0.85, // Target 85% of memory limit
|
||||
DerivativeFilterAlpha: 0.2, // Strong filtering (20% new, 80% old)
|
||||
IntegralMax: 10.0, // Anti-windup: max 10 seconds accumulated
|
||||
IntegralMin: -2.0, // Allow small negative for faster recovery
|
||||
OutputMin: 0.0, // No delay minimum
|
||||
OutputMax: 1.0, // Max 1 second delay per write
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultPIDControllerForReads creates a PID controller tuned for read operations.
|
||||
// Reads should be more responsive but with less aggressive throttling.
|
||||
func DefaultPIDControllerForReads() *PIDController {
|
||||
return &PIDController{
|
||||
Kp: 0.3, // Lower proportional (reads are more important)
|
||||
Ki: 0.05, // Lower integral (don't accumulate as aggressively)
|
||||
Kd: 0.02, // Very small derivative
|
||||
Setpoint: 0.90, // Target 90% (more tolerant of memory use)
|
||||
DerivativeFilterAlpha: 0.15, // Very strong filtering
|
||||
IntegralMax: 5.0, // Lower anti-windup limit
|
||||
IntegralMin: -1.0, // Allow small negative
|
||||
OutputMin: 0.0, // No delay minimum
|
||||
OutputMax: 0.5, // Max 500ms delay per read
|
||||
}
|
||||
}
|
||||
|
||||
// NewPIDController creates a new PID controller with custom parameters.
|
||||
func NewPIDController(
|
||||
kp, ki, kd float64,
|
||||
setpoint float64,
|
||||
derivativeFilterAlpha float64,
|
||||
integralMin, integralMax float64,
|
||||
outputMin, outputMax float64,
|
||||
) *PIDController {
|
||||
return &PIDController{
|
||||
Kp: kp,
|
||||
Ki: ki,
|
||||
Kd: kd,
|
||||
Setpoint: setpoint,
|
||||
DerivativeFilterAlpha: derivativeFilterAlpha,
|
||||
IntegralMin: integralMin,
|
||||
IntegralMax: integralMax,
|
||||
OutputMin: outputMin,
|
||||
OutputMax: outputMax,
|
||||
}
|
||||
}
|
||||
|
||||
// Update computes the PID output based on the current process variable.
|
||||
// The process variable should be in the range [0.0, 1.0+] representing load level.
|
||||
//
|
||||
// Returns the recommended delay in seconds. A value of 0 means no delay needed.
|
||||
func (p *PIDController) Update(processVariable float64) float64 {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Initialize on first call
|
||||
if !p.initialized {
|
||||
p.lastUpdate = now
|
||||
p.prevError = processVariable - p.Setpoint
|
||||
p.prevFilteredError = p.prevError
|
||||
p.initialized = true
|
||||
return 0 // No delay on first call
|
||||
}
|
||||
|
||||
// Calculate time delta
|
||||
dt := now.Sub(p.lastUpdate).Seconds()
|
||||
if dt <= 0 {
|
||||
dt = 0.001 // Minimum 1ms to avoid division by zero
|
||||
}
|
||||
p.lastUpdate = now
|
||||
|
||||
// Calculate current error (positive when above setpoint = need to throttle)
|
||||
error := processVariable - p.Setpoint
|
||||
|
||||
// Proportional term: immediate response to current error
|
||||
pTerm := p.Kp * error
|
||||
|
||||
// Integral term: accumulate error over time
|
||||
// Apply anti-windup by clamping the integral
|
||||
p.integral += error * dt
|
||||
p.integral = clamp(p.integral, p.IntegralMin, p.IntegralMax)
|
||||
iTerm := p.Ki * p.integral
|
||||
|
||||
// Derivative term with low-pass filter
|
||||
// Apply exponential moving average to filter high-frequency noise:
|
||||
// filtered = alpha * new + (1 - alpha) * old
|
||||
// This is equivalent to a first-order low-pass filter
|
||||
filteredError := p.DerivativeFilterAlpha*error + (1-p.DerivativeFilterAlpha)*p.prevFilteredError
|
||||
|
||||
// Derivative of the filtered error
|
||||
var dTerm float64
|
||||
if dt > 0 {
|
||||
dTerm = p.Kd * (filteredError - p.prevFilteredError) / dt
|
||||
}
|
||||
|
||||
// Update previous values for next iteration
|
||||
p.prevError = error
|
||||
p.prevFilteredError = filteredError
|
||||
|
||||
// Compute total output and clamp to limits
|
||||
output := pTerm + iTerm + dTerm
|
||||
output = clamp(output, p.OutputMin, p.OutputMax)
|
||||
|
||||
// Only return positive delays (throttle when above setpoint)
|
||||
if output < 0 {
|
||||
return 0
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
// Reset clears the controller state, useful when conditions change significantly.
|
||||
func (p *PIDController) Reset() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.integral = 0
|
||||
p.prevError = 0
|
||||
p.prevFilteredError = 0
|
||||
p.initialized = false
|
||||
}
|
||||
|
||||
// SetSetpoint updates the target setpoint.
|
||||
func (p *PIDController) SetSetpoint(setpoint float64) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.Setpoint = setpoint
|
||||
}
|
||||
|
||||
// SetGains updates the PID gains.
|
||||
func (p *PIDController) SetGains(kp, ki, kd float64) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.Kp = kp
|
||||
p.Ki = ki
|
||||
p.Kd = kd
|
||||
}
|
||||
|
||||
// GetState returns the current internal state for monitoring/debugging.
|
||||
func (p *PIDController) GetState() (integral, prevError, prevFilteredError float64) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.integral, p.prevError, p.prevFilteredError
|
||||
}
|
||||
|
||||
// clamp restricts a value to the range [min, max].
|
||||
func clamp(value, min, max float64) float64 {
|
||||
if math.IsNaN(value) {
|
||||
return 0
|
||||
}
|
||||
if value < min {
|
||||
return min
|
||||
}
|
||||
if value > max {
|
||||
return max
|
||||
}
|
||||
return value
|
||||
}
|
||||
176
pkg/ratelimit/pid_test.go
Normal file
176
pkg/ratelimit/pid_test.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPIDController_BasicOperation(t *testing.T) {
|
||||
pid := DefaultPIDControllerForWrites()
|
||||
|
||||
// First call should return 0 (initialization)
|
||||
delay := pid.Update(0.5)
|
||||
if delay != 0 {
|
||||
t.Errorf("expected 0 delay on first call, got %v", delay)
|
||||
}
|
||||
|
||||
// Sleep a bit to ensure dt > 0
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Process variable below setpoint (0.5 < 0.85) should return 0 delay
|
||||
delay = pid.Update(0.5)
|
||||
if delay != 0 {
|
||||
t.Errorf("expected 0 delay when below setpoint, got %v", delay)
|
||||
}
|
||||
|
||||
// Process variable above setpoint should return positive delay
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
delay = pid.Update(0.95) // 0.95 > 0.85 setpoint
|
||||
if delay <= 0 {
|
||||
t.Errorf("expected positive delay when above setpoint, got %v", delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPIDController_IntegralAccumulation(t *testing.T) {
|
||||
pid := NewPIDController(
|
||||
0.5, 0.5, 0.0, // High Ki, no Kd
|
||||
0.5, // setpoint
|
||||
0.2, // filter alpha
|
||||
-10, 10, // integral bounds
|
||||
0, 1.0, // output bounds
|
||||
)
|
||||
|
||||
// Initialize
|
||||
pid.Update(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Continuously above setpoint should accumulate integral
|
||||
for i := 0; i < 10; i++ {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
pid.Update(0.8) // 0.3 above setpoint
|
||||
}
|
||||
|
||||
integral, _, _ := pid.GetState()
|
||||
if integral <= 0 {
|
||||
t.Errorf("expected positive integral after sustained error, got %v", integral)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPIDController_FilteredDerivative(t *testing.T) {
|
||||
pid := NewPIDController(
|
||||
0.0, 0.0, 1.0, // Only Kd
|
||||
0.5, // setpoint
|
||||
0.5, // 50% filtering
|
||||
-10, 10,
|
||||
0, 1.0,
|
||||
)
|
||||
|
||||
// Initialize with low value
|
||||
pid.Update(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Second call with same value - derivative should be near zero
|
||||
pid.Update(0.5)
|
||||
_, _, prevFiltered := pid.GetState()
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Big jump - filtered derivative should be dampened
|
||||
delay := pid.Update(1.0)
|
||||
|
||||
// The filtered derivative should cause some response, but dampened
|
||||
// Since we only have Kd=1.0 and alpha=0.5, the response should be modest
|
||||
if delay < 0 {
|
||||
t.Errorf("expected non-negative delay, got %v", delay)
|
||||
}
|
||||
|
||||
_, _, newFiltered := pid.GetState()
|
||||
// Filtered error should have moved toward the new error but not fully
|
||||
if newFiltered <= prevFiltered {
|
||||
t.Errorf("filtered error should increase with rising process variable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPIDController_AntiWindup(t *testing.T) {
|
||||
pid := NewPIDController(
|
||||
0.0, 1.0, 0.0, // Only Ki
|
||||
0.5, // setpoint
|
||||
0.2, // filter alpha
|
||||
-1.0, 1.0, // tight integral bounds
|
||||
0, 10.0, // wide output bounds
|
||||
)
|
||||
|
||||
// Initialize
|
||||
pid.Update(0.5)
|
||||
|
||||
// Drive the integral to its limit
|
||||
for i := 0; i < 100; i++ {
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
pid.Update(1.0) // Large positive error
|
||||
}
|
||||
|
||||
integral, _, _ := pid.GetState()
|
||||
if integral > 1.0 {
|
||||
t.Errorf("integral should be clamped at 1.0, got %v", integral)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPIDController_Reset(t *testing.T) {
|
||||
pid := DefaultPIDControllerForWrites()
|
||||
|
||||
// Build up some state
|
||||
pid.Update(0.5)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
pid.Update(0.9)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
pid.Update(0.95)
|
||||
|
||||
// Reset
|
||||
pid.Reset()
|
||||
|
||||
integral, prevErr, prevFiltered := pid.GetState()
|
||||
if integral != 0 || prevErr != 0 || prevFiltered != 0 {
|
||||
t.Errorf("expected all state to be zero after reset")
|
||||
}
|
||||
|
||||
// Next call should behave like first call
|
||||
delay := pid.Update(0.9)
|
||||
if delay != 0 {
|
||||
t.Errorf("expected 0 delay on first call after reset, got %v", delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPIDController_SetGains(t *testing.T) {
|
||||
pid := DefaultPIDControllerForWrites()
|
||||
|
||||
// Change gains
|
||||
pid.SetGains(1.0, 0.5, 0.1)
|
||||
|
||||
if pid.Kp != 1.0 || pid.Ki != 0.5 || pid.Kd != 0.1 {
|
||||
t.Errorf("gains not updated correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPIDController_SetSetpoint(t *testing.T) {
|
||||
pid := DefaultPIDControllerForWrites()
|
||||
|
||||
pid.SetSetpoint(0.7)
|
||||
|
||||
if pid.Setpoint != 0.7 {
|
||||
t.Errorf("setpoint not updated, got %v", pid.Setpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultControllers(t *testing.T) {
|
||||
writePID := DefaultPIDControllerForWrites()
|
||||
readPID := DefaultPIDControllerForReads()
|
||||
|
||||
// Write controller should have higher gains and lower setpoint
|
||||
if writePID.Kp <= readPID.Kp {
|
||||
t.Errorf("write Kp should be higher than read Kp")
|
||||
}
|
||||
|
||||
if writePID.Setpoint >= readPID.Setpoint {
|
||||
t.Errorf("write setpoint should be lower than read setpoint")
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"next.orly.dev/app/config"
|
||||
"next.orly.dev/pkg/acl"
|
||||
"next.orly.dev/pkg/database"
|
||||
"next.orly.dev/pkg/ratelimit"
|
||||
)
|
||||
|
||||
// Options configures relay startup behavior.
|
||||
@@ -126,8 +127,11 @@ func Start(cfg *config.C, opts *Options) (relay *Relay, err error) {
|
||||
}
|
||||
acl.Registry.Syncer()
|
||||
|
||||
// Create rate limiter (disabled for test relay instances)
|
||||
limiter := ratelimit.NewDisabledLimiter()
|
||||
|
||||
// Start the relay
|
||||
relay.quit = app.Run(relay.ctx, cfg, relay.db)
|
||||
relay.quit = app.Run(relay.ctx, cfg, relay.db, limiter)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
v0.34.6
|
||||
v0.34.7
|
||||
Reference in New Issue
Block a user