Add Blossom bandwidth limiting and tune rate limiters (v0.49.0)
Some checks failed
Go / build-and-release (push) Has been cancelled
Some checks failed
Go / build-and-release (push) Has been cancelled
- Add token-bucket bandwidth rate limiting for Blossom uploads - ORLY_BLOSSOM_RATE_LIMIT enables limiting (default: false) - ORLY_BLOSSOM_DAILY_LIMIT_MB sets daily limit (default: 10MB) - ORLY_BLOSSOM_BURST_LIMIT_MB sets burst cap (default: 50MB) - Followed users, admins, owners are exempt (unlimited) - Change emergency mode throttling from exponential to linear scaling - Old: 16x multiplier at emergency threshold entry - New: 1x at threshold, +1x per 20% excess pressure - Reduce follows ACL throttle increment from 200ms to 25ms per event - Update dependencies Files modified: - app/blossom.go: Pass rate limit config to blossom server - app/config/config.go: Add Blossom rate limit config options - pkg/blossom/ratelimit.go: New bandwidth limiter implementation - pkg/blossom/server.go: Add rate limiter integration - pkg/blossom/handlers.go: Check rate limits on upload/mirror/media - pkg/ratelimit/limiter.go: Linear emergency throttling - pkg/acl/follows.go: Reduce default throttle increment Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -200,6 +200,12 @@ func (s *Server) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check bandwidth rate limit (non-followed users)
|
||||
if !s.checkBandwidthLimit(pubkey, remoteAddr, int64(len(body))) {
|
||||
s.setErrorResponse(w, http.StatusTooManyRequests, "upload rate limit exceeded, try again later")
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate SHA256 after auth check
|
||||
sha256Hash := CalculateSHA256(body)
|
||||
sha256Hex := hex.Enc(sha256Hash)
|
||||
@@ -647,6 +653,12 @@ func (s *Server) handleMirror(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check bandwidth rate limit (non-followed users)
|
||||
if !s.checkBandwidthLimit(pubkey, remoteAddr, int64(len(body))) {
|
||||
s.setErrorResponse(w, http.StatusTooManyRequests, "upload rate limit exceeded, try again later")
|
||||
return
|
||||
}
|
||||
|
||||
// Note: pubkey may be nil for anonymous uploads if ACL allows it
|
||||
|
||||
// Detect MIME type from remote response
|
||||
@@ -726,6 +738,12 @@ func (s *Server) handleMediaUpload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check bandwidth rate limit (non-followed users)
|
||||
if !s.checkBandwidthLimit(pubkey, remoteAddr, int64(len(body))) {
|
||||
s.setErrorResponse(w, http.StatusTooManyRequests, "upload rate limit exceeded, try again later")
|
||||
return
|
||||
}
|
||||
|
||||
// Note: pubkey may be nil for anonymous uploads if ACL allows it
|
||||
|
||||
// Optimize media (placeholder - actual optimization would be implemented here)
|
||||
|
||||
131
pkg/blossom/ratelimit.go
Normal file
131
pkg/blossom/ratelimit.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package blossom
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BandwidthState tracks upload bandwidth for an identity
|
||||
type BandwidthState struct {
|
||||
BucketBytes int64 // Current token bucket level (bytes available)
|
||||
LastUpdate time.Time // Last time bucket was updated
|
||||
}
|
||||
|
||||
// BandwidthLimiter implements token bucket rate limiting for uploads.
|
||||
// Each identity gets a bucket that replenishes at dailyLimit/day rate.
|
||||
// Uploads consume tokens from the bucket.
|
||||
type BandwidthLimiter struct {
|
||||
mu sync.Mutex
|
||||
states map[string]*BandwidthState // keyed by pubkey hex or IP
|
||||
dailyLimit int64 // bytes per day
|
||||
burstLimit int64 // max bucket size (burst capacity)
|
||||
refillRate float64 // bytes per second refill rate
|
||||
}
|
||||
|
||||
// NewBandwidthLimiter creates a new bandwidth limiter.
|
||||
// dailyLimitMB is the average daily limit in megabytes.
|
||||
// burstLimitMB is the maximum burst capacity in megabytes.
|
||||
func NewBandwidthLimiter(dailyLimitMB, burstLimitMB int64) *BandwidthLimiter {
|
||||
dailyBytes := dailyLimitMB * 1024 * 1024
|
||||
burstBytes := burstLimitMB * 1024 * 1024
|
||||
|
||||
return &BandwidthLimiter{
|
||||
states: make(map[string]*BandwidthState),
|
||||
dailyLimit: dailyBytes,
|
||||
burstLimit: burstBytes,
|
||||
refillRate: float64(dailyBytes) / 86400.0, // bytes per second
|
||||
}
|
||||
}
|
||||
|
||||
// CheckAndConsume checks if an upload of the given size is allowed for the identity,
|
||||
// and if so, consumes the tokens. Returns true if allowed, false if rate limited.
|
||||
// The identity should be pubkey hex for authenticated users, or IP for anonymous.
|
||||
func (bl *BandwidthLimiter) CheckAndConsume(identity string, sizeBytes int64) bool {
|
||||
bl.mu.Lock()
|
||||
defer bl.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
state, exists := bl.states[identity]
|
||||
|
||||
if !exists {
|
||||
// New identity starts with full burst capacity
|
||||
state = &BandwidthState{
|
||||
BucketBytes: bl.burstLimit,
|
||||
LastUpdate: now,
|
||||
}
|
||||
bl.states[identity] = state
|
||||
} else {
|
||||
// Refill bucket based on elapsed time
|
||||
elapsed := now.Sub(state.LastUpdate).Seconds()
|
||||
refill := int64(elapsed * bl.refillRate)
|
||||
state.BucketBytes += refill
|
||||
if state.BucketBytes > bl.burstLimit {
|
||||
state.BucketBytes = bl.burstLimit
|
||||
}
|
||||
state.LastUpdate = now
|
||||
}
|
||||
|
||||
// Check if upload fits in bucket
|
||||
if state.BucketBytes >= sizeBytes {
|
||||
state.BucketBytes -= sizeBytes
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// GetAvailable returns the currently available bytes for an identity.
|
||||
func (bl *BandwidthLimiter) GetAvailable(identity string) int64 {
|
||||
bl.mu.Lock()
|
||||
defer bl.mu.Unlock()
|
||||
|
||||
state, exists := bl.states[identity]
|
||||
if !exists {
|
||||
return bl.burstLimit // New users have full capacity
|
||||
}
|
||||
|
||||
// Calculate current level with refill
|
||||
now := time.Now()
|
||||
elapsed := now.Sub(state.LastUpdate).Seconds()
|
||||
refill := int64(elapsed * bl.refillRate)
|
||||
available := state.BucketBytes + refill
|
||||
if available > bl.burstLimit {
|
||||
available = bl.burstLimit
|
||||
}
|
||||
|
||||
return available
|
||||
}
|
||||
|
||||
// GetTimeUntilAvailable returns how long until the given bytes will be available.
|
||||
func (bl *BandwidthLimiter) GetTimeUntilAvailable(identity string, sizeBytes int64) time.Duration {
|
||||
available := bl.GetAvailable(identity)
|
||||
if available >= sizeBytes {
|
||||
return 0
|
||||
}
|
||||
|
||||
needed := sizeBytes - available
|
||||
seconds := float64(needed) / bl.refillRate
|
||||
return time.Duration(seconds * float64(time.Second))
|
||||
}
|
||||
|
||||
// Cleanup removes entries that have fully replenished (at burst limit).
|
||||
func (bl *BandwidthLimiter) Cleanup() {
|
||||
bl.mu.Lock()
|
||||
defer bl.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
for key, state := range bl.states {
|
||||
elapsed := now.Sub(state.LastUpdate).Seconds()
|
||||
refill := int64(elapsed * bl.refillRate)
|
||||
if state.BucketBytes+refill >= bl.burstLimit {
|
||||
delete(bl.states, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stats returns the number of tracked identities.
|
||||
func (bl *BandwidthLimiter) Stats() int {
|
||||
bl.mu.Lock()
|
||||
defer bl.mu.Unlock()
|
||||
return len(bl.states)
|
||||
}
|
||||
@@ -19,6 +19,9 @@ type Server struct {
|
||||
maxBlobSize int64
|
||||
allowedMimeTypes map[string]bool
|
||||
requireAuth bool
|
||||
|
||||
// Rate limiting for uploads
|
||||
bandwidthLimiter *BandwidthLimiter
|
||||
}
|
||||
|
||||
// Config holds configuration for the Blossom server
|
||||
@@ -27,6 +30,11 @@ type Config struct {
|
||||
MaxBlobSize int64
|
||||
AllowedMimeTypes []string
|
||||
RequireAuth bool
|
||||
|
||||
// Rate limiting (for non-followed users)
|
||||
RateLimitEnabled bool
|
||||
DailyLimitMB int64
|
||||
BurstLimitMB int64
|
||||
}
|
||||
|
||||
// NewServer creates a new Blossom server instance
|
||||
@@ -48,6 +56,20 @@ func NewServer(db *database.D, aclRegistry *acl.S, cfg *Config) *Server {
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize bandwidth limiter if enabled
|
||||
var bwLimiter *BandwidthLimiter
|
||||
if cfg.RateLimitEnabled {
|
||||
dailyMB := cfg.DailyLimitMB
|
||||
if dailyMB <= 0 {
|
||||
dailyMB = 10 // 10MB default
|
||||
}
|
||||
burstMB := cfg.BurstLimitMB
|
||||
if burstMB <= 0 {
|
||||
burstMB = 50 // 50MB default burst
|
||||
}
|
||||
bwLimiter = NewBandwidthLimiter(dailyMB, burstMB)
|
||||
}
|
||||
|
||||
return &Server{
|
||||
db: db,
|
||||
storage: storage,
|
||||
@@ -56,6 +78,7 @@ func NewServer(db *database.D, aclRegistry *acl.S, cfg *Config) *Server {
|
||||
maxBlobSize: cfg.MaxBlobSize,
|
||||
allowedMimeTypes: allowedMap,
|
||||
requireAuth: cfg.RequireAuth,
|
||||
bandwidthLimiter: bwLimiter,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,6 +231,44 @@ func (s *Server) checkACL(
|
||||
return actual >= required
|
||||
}
|
||||
|
||||
// isRateLimitExempt returns true if the user is exempt from rate limiting.
|
||||
// Users with write access or higher (followed users, admins, owners) are exempt.
|
||||
func (s *Server) isRateLimitExempt(pubkey []byte, remoteAddr string) bool {
|
||||
if s.acl == nil {
|
||||
return true // No ACL configured, no rate limiting
|
||||
}
|
||||
|
||||
level := s.acl.GetAccessLevel(pubkey, remoteAddr)
|
||||
|
||||
// Followed users get "write" level, admins/owners get higher
|
||||
// Only "read" and "none" are rate limited
|
||||
return level == "write" || level == "admin" || level == "owner"
|
||||
}
|
||||
|
||||
// checkBandwidthLimit checks if the upload is allowed under rate limits.
|
||||
// Returns true if allowed, false if rate limited.
|
||||
// Exempt users (followed, admin, owner) always return true.
|
||||
func (s *Server) checkBandwidthLimit(pubkey []byte, remoteAddr string, sizeBytes int64) bool {
|
||||
if s.bandwidthLimiter == nil {
|
||||
return true // No rate limiting configured
|
||||
}
|
||||
|
||||
// Check if user is exempt
|
||||
if s.isRateLimitExempt(pubkey, remoteAddr) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Use pubkey hex if available, otherwise IP
|
||||
var identity string
|
||||
if len(pubkey) > 0 {
|
||||
identity = string(pubkey) // Will be converted to hex in handler
|
||||
} else {
|
||||
identity = remoteAddr
|
||||
}
|
||||
|
||||
return s.bandwidthLimiter.CheckAndConsume(identity, sizeBytes)
|
||||
}
|
||||
|
||||
// BaseURLKey is the context key for the base URL (exported for use by app handler)
|
||||
type BaseURLKey struct{}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user