enhance spider with rate limit handling, follow list updates, and improved reconnect logic; bump version to v0.29.0

also reduces CPU load for spider, and minor CORS fixes

.claude/settings.local.json

@@ -48,7 +48,18 @@
       "Bash(./test-policy.sh:*)",
       "Bash(docker rm:*)",
       "Bash(./scripts/docker-policy/test-policy.sh:*)",
-      "Bash(./policytest:*)"
+      "Bash(./policytest:*)",
+      "WebSearch",
+      "WebFetch(domain:blog.scottlogic.com)",
+      "WebFetch(domain:eli.thegreenplace.net)",
+      "WebFetch(domain:learn-wasm.dev)",
+      "Bash(curl:*)",
+      "Bash(./build.sh)",
+      "Bash(./pkg/wasm/shell/run.sh:*)",
+      "Bash(./run.sh echo.wasm)",
+      "Bash(./test.sh)",
+      "Bash(ORLY_PPROF=cpu ORLY_LOG_LEVEL=info ORLY_LISTEN=0.0.0.0 ORLY_PORT=3334 ORLY_ADMINS=npub1fjqqy4a93z5zsjwsfxqhc2764kvykfdyttvldkkkdera8dr78vhsmmleku ORLY_OWNERS=npub1fjqqy4a93z5zsjwsfxqhc2764kvykfdyttvldkkkdera8dr78vhsmmleku ORLY_ACL_MODE=follows ORLY_SPIDER_MODE=follows timeout 120 go run:*)",
+      "Bash(go tool pprof:*)"
     ],
     "deny": [],
     "ask": []

app/main.go

@@ -122,6 +122,21 @@ func Run(
 				log.E.F("failed to start spider manager: %v", err)
 			} else {
 				log.I.F("spider manager started successfully in '%s' mode", cfg.SpiderMode)
+
+				// Hook up follow list update notifications from ACL to spider
+				if cfg.SpiderMode == "follows" {
+					for _, aclInstance := range acl.Registry.ACL {
+						if aclInstance.Type() == "follows" {
+							if follows, ok := aclInstance.(*acl.Follows); ok {
+								follows.SetFollowListUpdateCallback(func() {
+									log.I.F("follow list updated, notifying spider")
+									l.spiderManager.NotifyFollowListUpdate()
+								})
+								log.I.F("spider: follow list update notifications configured")
+							}
+						}
+					}
+				}
 			}
 		}
 	}

app/server.go

@@ -17,6 +17,7 @@ import (
 	"lol.mleku.dev/chk"
 	"next.orly.dev/app/config"
 	"next.orly.dev/pkg/acl"
+	"next.orly.dev/pkg/blossom"
 	"next.orly.dev/pkg/database"
 	"next.orly.dev/pkg/encoders/event"
 	"next.orly.dev/pkg/encoders/filter"
@@ -29,7 +30,6 @@ import (
 	"next.orly.dev/pkg/protocol/publish"
 	"next.orly.dev/pkg/spider"
 	dsync "next.orly.dev/pkg/sync"
-	blossom "next.orly.dev/pkg/blossom"
 )
 
 type Server struct {
@@ -91,19 +91,9 @@ func (s *Server) isIPBlacklisted(remote string) bool {
 }
 
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Set comprehensive CORS headers for proxy compatibility
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-	w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
-	w.Header().Set("Access-Control-Allow-Headers",
-		"Origin, X-Requested-With, Content-Type, Accept, Authorization, "+
-			"X-Forwarded-For, X-Forwarded-Proto, X-Forwarded-Host, X-Real-IP, "+
-			"Upgrade, Connection, Sec-WebSocket-Key, Sec-WebSocket-Version, "+
-			"Sec-WebSocket-Protocol, Sec-WebSocket-Extensions")
-	w.Header().Set("Access-Control-Allow-Credentials", "true")
-	w.Header().Set("Access-Control-Max-Age", "86400")
-
-	// Add proxy-friendly headers
-	w.Header().Set("Vary", "Origin, Access-Control-Request-Method, Access-Control-Request-Headers")
+	// CORS headers should be handled by the reverse proxy (Caddy/nginx)
+	// to avoid duplicate headers. If running without a reverse proxy,
+	// uncomment the CORS configuration below or configure via environment variable.
 
 	// Handle preflight OPTIONS requests
 	if r.Method == "OPTIONS" {
@@ -245,7 +235,9 @@ func (s *Server) UserInterface() {
 	s.mux.HandleFunc("/api/sprocket/update", s.handleSprocketUpdate)
 	s.mux.HandleFunc("/api/sprocket/restart", s.handleSprocketRestart)
 	s.mux.HandleFunc("/api/sprocket/versions", s.handleSprocketVersions)
-	s.mux.HandleFunc("/api/sprocket/delete-version", s.handleSprocketDeleteVersion)
+	s.mux.HandleFunc(
+		"/api/sprocket/delete-version", s.handleSprocketDeleteVersion,
+	)
 	s.mux.HandleFunc("/api/sprocket/config", s.handleSprocketConfig)
 	// NIP-86 management endpoint
 	s.mux.HandleFunc("/api/nip86", s.handleNIP86Management)
@@ -343,7 +335,9 @@ func (s *Server) handleAuthChallenge(w http.ResponseWriter, r *http.Request) {
 
 	jsonData, err := json.Marshal(response)
 	if chk.E(err) {
-		http.Error(w, "Error generating challenge", http.StatusInternalServerError)
+		http.Error(
+			w, "Error generating challenge", http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -561,7 +555,10 @@ func (s *Server) handleExport(w http.ResponseWriter, r *http.Request) {
 	// Check permissions - require write, admin, or owner level
 	accessLevel := acl.Registry.GetAccessLevel(pubkey, r.RemoteAddr)
 	if accessLevel != "write" && accessLevel != "admin" && accessLevel != "owner" {
-		http.Error(w, "Write, admin, or owner permission required", http.StatusForbidden)
+		http.Error(
+			w, "Write, admin, or owner permission required",
+			http.StatusForbidden,
+		)
 		return
 	}
 
@@ -610,7 +607,9 @@ func (s *Server) handleExport(w http.ResponseWriter, r *http.Request) {
 	}
 
 	w.Header().Set("Content-Type", "application/x-ndjson")
-	w.Header().Set("Content-Disposition", "attachment; filename=\""+filename+"\"")
+	w.Header().Set(
+		"Content-Disposition", "attachment; filename=\""+filename+"\"",
+	)
 
 	// Stream export
 	s.D.Export(s.Ctx, w, pks...)
@@ -725,7 +724,9 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
 	// Check permissions - require admin or owner level
 	accessLevel := acl.Registry.GetAccessLevel(pubkey, r.RemoteAddr)
 	if accessLevel != "admin" && accessLevel != "owner" {
-		http.Error(w, "Admin or owner permission required", http.StatusForbidden)
+		http.Error(
+			w, "Admin or owner permission required", http.StatusForbidden,
+		)
 		return
 	}
 
@@ -785,7 +786,9 @@ func (s *Server) handleSprocketStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	jsonData, err := json.Marshal(status)
 	if chk.E(err) {
-		http.Error(w, "Error generating response", http.StatusInternalServerError)
+		http.Error(
+			w, "Error generating response", http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -826,7 +829,10 @@ func (s *Server) handleSprocketUpdate(w http.ResponseWriter, r *http.Request) {
 
 	// Update the sprocket script
 	if err := s.sprocketManager.UpdateSprocket(string(body)); chk.E(err) {
-		http.Error(w, fmt.Sprintf("Failed to update sprocket: %v", err), http.StatusInternalServerError)
+		http.Error(
+			w, fmt.Sprintf("Failed to update sprocket: %v", err),
+			http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -861,7 +867,10 @@ func (s *Server) handleSprocketRestart(w http.ResponseWriter, r *http.Request) {
 
 	// Restart the sprocket script
 	if err := s.sprocketManager.RestartSprocket(); chk.E(err) {
-		http.Error(w, fmt.Sprintf("Failed to restart sprocket: %v", err), http.StatusInternalServerError)
+		http.Error(
+			w, fmt.Sprintf("Failed to restart sprocket: %v", err),
+			http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -870,7 +879,9 @@ func (s *Server) handleSprocketRestart(w http.ResponseWriter, r *http.Request) {
 }
 
 // handleSprocketVersions returns all sprocket script versions
-func (s *Server) handleSprocketVersions(w http.ResponseWriter, r *http.Request) {
+func (s *Server) handleSprocketVersions(
+	w http.ResponseWriter, r *http.Request,
+) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 		return
@@ -896,14 +907,19 @@ func (s *Server) handleSprocketVersions(w http.ResponseWriter, r *http.Request)
 
 	versions, err := s.sprocketManager.GetSprocketVersions()
 	if chk.E(err) {
-		http.Error(w, fmt.Sprintf("Failed to get sprocket versions: %v", err), http.StatusInternalServerError)
+		http.Error(
+			w, fmt.Sprintf("Failed to get sprocket versions: %v", err),
+			http.StatusInternalServerError,
+		)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/json")
 	jsonData, err := json.Marshal(versions)
 	if chk.E(err) {
-		http.Error(w, "Error generating response", http.StatusInternalServerError)
+		http.Error(
+			w, "Error generating response", http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -911,7 +927,9 @@ func (s *Server) handleSprocketVersions(w http.ResponseWriter, r *http.Request)
 }
 
 // handleSprocketDeleteVersion deletes a specific sprocket version
-func (s *Server) handleSprocketDeleteVersion(w http.ResponseWriter, r *http.Request) {
+func (s *Server) handleSprocketDeleteVersion(
+	w http.ResponseWriter, r *http.Request,
+) {
 	if r.Method != http.MethodPost {
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 		return
@@ -957,7 +975,10 @@ func (s *Server) handleSprocketDeleteVersion(w http.ResponseWriter, r *http.Requ
 
 	// Delete the sprocket version
 	if err := s.sprocketManager.DeleteSprocketVersion(request.Filename); chk.E(err) {
-		http.Error(w, fmt.Sprintf("Failed to delete sprocket version: %v", err), http.StatusInternalServerError)
+		http.Error(
+			w, fmt.Sprintf("Failed to delete sprocket version: %v", err),
+			http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -982,7 +1003,9 @@ func (s *Server) handleSprocketConfig(w http.ResponseWriter, r *http.Request) {
 
 	jsonData, err := json.Marshal(response)
 	if chk.E(err) {
-		http.Error(w, "Error generating response", http.StatusInternalServerError)
+		http.Error(
+			w, "Error generating response", http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -1006,7 +1029,9 @@ func (s *Server) handleACLMode(w http.ResponseWriter, r *http.Request) {
 
 	jsonData, err := json.Marshal(response)
 	if chk.E(err) {
-		http.Error(w, "Error generating response", http.StatusInternalServerError)
+		http.Error(
+			w, "Error generating response", http.StatusInternalServerError,
+		)
 		return
 	}
 
@@ -1016,7 +1041,9 @@ func (s *Server) handleACLMode(w http.ResponseWriter, r *http.Request) {
 // handleSyncCurrent handles requests for the current serial number
 func (s *Server) handleSyncCurrent(w http.ResponseWriter, r *http.Request) {
 	if s.syncManager == nil {
-		http.Error(w, "Sync manager not initialized", http.StatusServiceUnavailable)
+		http.Error(
+			w, "Sync manager not initialized", http.StatusServiceUnavailable,
+		)
 		return
 	}
 
@@ -1031,7 +1058,9 @@ func (s *Server) handleSyncCurrent(w http.ResponseWriter, r *http.Request) {
 // handleSyncEventIDs handles requests for event IDs with their serial numbers
 func (s *Server) handleSyncEventIDs(w http.ResponseWriter, r *http.Request) {
 	if s.syncManager == nil {
-		http.Error(w, "Sync manager not initialized", http.StatusServiceUnavailable)
+		http.Error(
+			w, "Sync manager not initialized", http.StatusServiceUnavailable,
+		)
 		return
 	}
 
@@ -1044,12 +1073,16 @@ func (s *Server) handleSyncEventIDs(w http.ResponseWriter, r *http.Request) {
 }
 
 // validatePeerRequest validates NIP-98 authentication and checks if the requesting peer is authorized
-func (s *Server) validatePeerRequest(w http.ResponseWriter, r *http.Request) bool {
+func (s *Server) validatePeerRequest(
+	w http.ResponseWriter, r *http.Request,
+) bool {
 	// Validate NIP-98 authentication
 	valid, pubkey, err := httpauth.CheckAuth(r)
 	if err != nil {
 		log.Printf("NIP-98 auth validation error: %v", err)
-		http.Error(w, "Authentication validation failed", http.StatusUnauthorized)
+		http.Error(
+			w, "Authentication validation failed", http.StatusUnauthorized,
+		)
 		return false
 	}
 	if !valid {

contrib/stella/APACHE-PROXY-GUIDE.md

@@ -27,7 +27,7 @@ docker run -d \
   -v /data/orly-relay:/data \
   -e ORLY_OWNERS=npub1v30tsz9vw6ylpz63g0a702nj3xa26t3m7p5us8f2y2sd8v6cnsvq465zjx \
   -e ORLY_ADMINS=npub1v30tsz9vw6ylpz63g0a702nj3xa26t3m7p5us8f2y2sd8v6cnsvq465zjx,npub1l5sga6xg72phsz5422ykujprejwud075ggrr3z2hwyrfgr7eylqstegx9z,npub1m4ny6hjqzepn4rxknuq94c2gpqzr29ufkkw7ttcxyak7v43n6vvsajc2jl \
-  -e ORLY_BOOTSTRAP_RELAYS=wss://profiles.nostr1.com,wss://purplepag.es,wss://relay.nostr.band,wss://relay.damus.io \
+  -e ORLY_BOOTSTRAP_RELAYS=wss://profiles.nostr1.com,wss://purplepag.es,wss://relay.damus.io \
   -e ORLY_RELAY_URL=wss://orly-relay.imwald.eu \
   -e ORLY_ACL_MODE=follows \
   -e ORLY_SUBSCRIPTION_ENABLED=false \

contrib/stella/docker-compose.yml

@@ -28,7 +28,7 @@ services:
       - ORLY_ACL_MODE=follows
 
       # Bootstrap relay URLs for initial sync
-      - ORLY_BOOTSTRAP_RELAYS=wss://profiles.nostr1.com,wss://purplepag.es,wss://relay.nostr.band,wss://relay.damus.io
+      - ORLY_BOOTSTRAP_RELAYS=wss://profiles.nostr1.com,wss://purplepag.es,wss://relay.damus.io
 
       # Subscription Settings (optional)
       - ORLY_SUBSCRIPTION_ENABLED=false

pkg/acl/follows.go

@@ -46,6 +46,8 @@ type Follows struct {
 	subsCancel context.CancelFunc
 	// Track last follow list fetch time
 	lastFollowListFetch time.Time
+	// Callback for external notification of follow list changes
+	onFollowListUpdate func()
 }
 
 func (f *Follows) Configure(cfg ...any) (err error) {
@@ -314,7 +316,6 @@ func (f *Follows) adminRelays() (urls []string) {
 				"wss://nostr.wine",
 				"wss://nos.lol",
 				"wss://relay.damus.io",
-				"wss://nostr.band",
 			}
 			log.I.F("using failover relays: %v", failoverRelays)
 			for _, relay := range failoverRelays {
@@ -933,6 +934,13 @@ func (f *Follows) AdminRelays() []string {
 	return f.adminRelays()
 }
 
+// SetFollowListUpdateCallback sets a callback to be called when the follow list is updated
+func (f *Follows) SetFollowListUpdateCallback(callback func()) {
+	f.followsMx.Lock()
+	defer f.followsMx.Unlock()
+	f.onFollowListUpdate = callback
+}
+
 // AddFollow appends a pubkey to the in-memory follows list if not already present
 // and signals the syncer to refresh subscriptions.
 func (f *Follows) AddFollow(pub []byte) {
@@ -961,6 +969,10 @@ func (f *Follows) AddFollow(pub []byte) {
 			// if channel is full or not yet listened to, ignore
 		}
 	}
+	// notify external listeners (e.g., spider)
+	if f.onFollowListUpdate != nil {
+		go f.onFollowListUpdate()
+	}
 }
 
 func init() {

pkg/encoders/envelopes/reqenvelope/reqenvelope.go

@@ -6,7 +6,6 @@ import (
 	"io"
 
 	"lol.mleku.dev/chk"
-	"lol.mleku.dev/log"
 	"next.orly.dev/pkg/encoders/envelopes"
 	"next.orly.dev/pkg/encoders/filter"
 	"next.orly.dev/pkg/encoders/text"
@@ -86,24 +85,19 @@ func (en *T) Marshal(dst []byte) (b []byte) {
 // string is correctly unescaped by NIP-01 escaping rules.
 func (en *T) Unmarshal(b []byte) (r []byte, err error) {
 	r = b
-	log.I.F("%s", r)
 	if en.Subscription, r, err = text.UnmarshalQuoted(r); chk.E(err) {
 		return
 	}
-	log.I.F("%s", r)
 	if r, err = text.Comma(r); chk.E(err) {
 		return
 	}
-	log.I.F("%s", r)
 	en.Filters = new(filter.S)
 	if r, err = en.Filters.Unmarshal(r); chk.E(err) {
 		return
 	}
-	log.I.F("%s", r)
 	if r, err = envelopes.SkipToTheEnd(r); chk.E(err) {
 		return
 	}
-	log.I.F("%s", r)
 	return
 }
 

pkg/protocol/ws/client.go

@@ -111,6 +111,7 @@ type RelayOption interface {
 var (
 	_ RelayOption = (WithCustomHandler)(nil)
 	_ RelayOption = (WithRequestHeader)(nil)
+	_ RelayOption = (WithNoticeHandler)(nil)
 )
 
 // WithCustomHandler must be a function that handles any relay message that couldn't be
@@ -128,6 +129,18 @@ func (ch WithRequestHeader) ApplyRelayOption(r *Client) {
 	r.requestHeader = http.Header(ch)
 }
 
+// WithNoticeHandler must be a function that handles NOTICE messages from the relay.
+type WithNoticeHandler func(notice []byte)
+
+func (nh WithNoticeHandler) ApplyRelayOption(r *Client) {
+	r.notices = make(chan []byte, 8)
+	go func() {
+		for notice := range r.notices {
+			nh(notice)
+		}
+	}()
+}
+
 // String just returns the relay URL.
 func (r *Client) String() string {
 	return r.URL

pkg/spider/spider.go

@@ -3,6 +3,7 @@ package spider
 import (
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -23,12 +24,24 @@ const (
 	BatchSize = 20
 	// CatchupWindow is the extra time added to disconnection periods for catch-up
 	CatchupWindow = 30 * time.Minute
-	// ReconnectDelay is the delay between reconnection attempts
-	ReconnectDelay = 5 * time.Second
-	// MaxReconnectDelay is the maximum delay between reconnection attempts
-	MaxReconnectDelay = 5 * time.Minute
-	// BlackoutPeriod is the duration to blacklist a relay after MaxReconnectDelay is reached
+	// ReconnectDelay is the initial delay between reconnection attempts
+	ReconnectDelay = 10 * time.Second
+	// MaxReconnectDelay is the maximum delay before switching to blackout
+	MaxReconnectDelay = 1 * time.Hour
+	// BlackoutPeriod is the duration to blacklist a relay after max backoff is reached
 	BlackoutPeriod = 24 * time.Hour
+	// BatchCreationDelay is the delay between creating each batch subscription
+	BatchCreationDelay = 500 * time.Millisecond
+	// RateLimitBackoffDuration is how long to wait when we get a rate limit error
+	RateLimitBackoffDuration = 1 * time.Minute
+	// RateLimitBackoffMultiplier is the factor by which we increase backoff on repeated rate limits
+	RateLimitBackoffMultiplier = 2
+	// MaxRateLimitBackoff is the maximum backoff duration for rate limiting
+	MaxRateLimitBackoff = 30 * time.Minute
+	// MainLoopInterval is how often the spider checks for updates
+	MainLoopInterval = 5 * time.Minute
+	// EventHandlerBufferSize is the buffer size for event channels
+	EventHandlerBufferSize = 100
 )
 
 // Spider manages connections to admin relays and syncs events for followed pubkeys
@@ -51,6 +64,9 @@ type Spider struct {
 	// Callbacks for getting updated data
 	getAdminRelays func() []string
 	getFollowList  func() [][]byte
+
+	// Notification channel for follow list updates
+	followListUpdated chan struct{}
 }
 
 // RelayConnection manages a single relay connection and its subscriptions
@@ -72,6 +88,10 @@ type RelayConnection struct {
 
 	// Blackout tracking for IP filters
 	blackoutUntil time.Time
+
+	// Rate limiting tracking
+	rateLimitBackoff time.Duration
+	rateLimitUntil   time.Time
 }
 
 // BatchSubscription represents a subscription for a batch of pubkeys
@@ -110,12 +130,13 @@ func New(ctx context.Context, db *database.D, pub publisher.I, mode string) (s *
 
 	ctx, cancel := context.WithCancel(ctx)
 	s = &Spider{
-		ctx:         ctx,
-		cancel:      cancel,
-		db:          db,
-		pub:         pub,
-		mode:        mode,
-		connections: make(map[string]*RelayConnection),
+		ctx:               ctx,
+		cancel:            cancel,
+		db:                db,
+		pub:               pub,
+		mode:              mode,
+		connections:       make(map[string]*RelayConnection),
+		followListUpdated: make(chan struct{}, 1),
 	}
 
 	return
@@ -129,6 +150,19 @@ func (s *Spider) SetCallbacks(getAdminRelays func() []string, getFollowList func
 	s.getFollowList = getFollowList
 }
 
+// NotifyFollowListUpdate signals the spider that the follow list has been updated
+func (s *Spider) NotifyFollowListUpdate() {
+	if s.followListUpdated != nil {
+		select {
+		case s.followListUpdated <- struct{}{}:
+			log.D.F("spider: follow list update notification sent")
+		default:
+			// Channel full, update already pending
+			log.D.F("spider: follow list update notification already pending")
+		}
+	}
+}
+
 // Start begins the spider operation
 func (s *Spider) Start() (err error) {
 	s.mu.Lock()
@@ -182,14 +216,20 @@ func (s *Spider) Stop() {
 
 // mainLoop is the main spider loop that manages connections and subscriptions
 func (s *Spider) mainLoop() {
-	ticker := time.NewTicker(30 * time.Second) // Check every 30 seconds
+	ticker := time.NewTicker(MainLoopInterval)
 	defer ticker.Stop()
 
+	log.I.F("spider: main loop started, checking every %v", MainLoopInterval)
+
 	for {
 		select {
 		case <-s.ctx.Done():
 			return
+		case <-s.followListUpdated:
+			log.I.F("spider: follow list updated, refreshing connections")
+			s.updateConnections()
 		case <-ticker.C:
+			log.D.F("spider: periodic check triggered")
 			s.updateConnections()
 		}
 	}
@@ -261,19 +301,24 @@ func (s *Spider) createConnection(url string, followList [][]byte) {
 // manage handles the lifecycle of a relay connection
 func (rc *RelayConnection) manage(followList [][]byte) {
 	for {
+		// Check context first
 		select {
 		case <-rc.ctx.Done():
+			log.D.F("spider: connection manager for %s stopping (context done)", rc.url)
 			return
 		default:
 		}
 
 		// Check if relay is blacked out
 		if rc.isBlackedOut() {
-			log.D.F("spider: %s is blacked out until %v", rc.url, rc.blackoutUntil)
+			waitDuration := time.Until(rc.blackoutUntil)
+			log.I.F("spider: %s is blacked out for %v more", rc.url, waitDuration)
+
+			// Wait for blackout to expire or context cancellation
 			select {
 			case <-rc.ctx.Done():
 				return
-			case <-time.After(time.Until(rc.blackoutUntil)):
+			case <-time.After(waitDuration):
 				// Blackout expired, reset delay and try again
 				rc.reconnectDelay = ReconnectDelay
 				log.I.F("spider: blackout period ended for %s, retrying", rc.url)
@@ -282,6 +327,7 @@ func (rc *RelayConnection) manage(followList [][]byte) {
 		}
 
 		// Attempt to connect
+		log.D.F("spider: attempting to connect to %s (backoff: %v)", rc.url, rc.reconnectDelay)
 		if err := rc.connect(); chk.E(err) {
 			log.W.F("spider: failed to connect to %s: %v", rc.url, err)
 			rc.waitBeforeReconnect()
@@ -290,8 +336,17 @@ func (rc *RelayConnection) manage(followList [][]byte) {
 
 		log.I.F("spider: connected to %s", rc.url)
 		rc.connectionStartTime = time.Now()
-		rc.reconnectDelay = ReconnectDelay // Reset delay on successful connection
-		rc.blackoutUntil = time.Time{}     // Clear blackout on successful connection
+
+		// Only reset reconnect delay on successful connection
+		// (don't reset if we had a quick disconnect before)
+		if rc.reconnectDelay > ReconnectDelay*8 {
+			// Gradual recovery: reduce by half instead of full reset
+			rc.reconnectDelay = rc.reconnectDelay / 2
+			log.D.F("spider: reducing backoff for %s to %v", rc.url, rc.reconnectDelay)
+		} else {
+			rc.reconnectDelay = ReconnectDelay
+		}
+		rc.blackoutUntil = time.Time{} // Clear blackout on successful connection
 
 		// Create subscriptions for follow list
 		rc.createSubscriptions(followList)
@@ -300,19 +355,25 @@ func (rc *RelayConnection) manage(followList [][]byte) {
 		<-rc.client.Context().Done()
 
 		log.W.F("spider: disconnected from %s: %v", rc.url, rc.client.ConnectionCause())
-		
-		// Check if disconnection happened very quickly (likely IP filter)
+
+		// Check if disconnection happened very quickly (likely IP filter or ban)
 		connectionDuration := time.Since(rc.connectionStartTime)
-		const quickDisconnectThreshold = 30 * time.Second
+		const quickDisconnectThreshold = 2 * time.Minute
 		if connectionDuration < quickDisconnectThreshold {
-			log.W.F("spider: quick disconnection from %s after %v (likely IP filter)", rc.url, connectionDuration)
-			// Don't reset the delay, keep the backoff
+			log.W.F("spider: quick disconnection from %s after %v (likely connection issue/ban)", rc.url, connectionDuration)
+			// Don't reset the delay, keep the backoff and increase it
 			rc.waitBeforeReconnect()
 		} else {
-			// Normal disconnection, reset backoff for future connections
-			rc.reconnectDelay = ReconnectDelay
+			// Normal disconnection after decent uptime - gentle backoff
+			log.I.F("spider: normal disconnection from %s after %v uptime", rc.url, connectionDuration)
+			// Small delay before reconnecting
+			select {
+			case <-rc.ctx.Done():
+				return
+			case <-time.After(5 * time.Second):
+			}
 		}
-		
+
 		rc.handleDisconnection()
 
 		// Clean up
@@ -326,15 +387,56 @@ func (rc *RelayConnection) connect() (err error) {
 	connectCtx, cancel := context.WithTimeout(rc.ctx, 10*time.Second)
 	defer cancel()
 
-	if rc.client, err = ws.RelayConnect(connectCtx, rc.url); chk.E(err) {
+	// Create client with notice handler to detect rate limiting
+	rc.client, err = ws.RelayConnect(connectCtx, rc.url, ws.WithNoticeHandler(rc.handleNotice))
+	if chk.E(err) {
 		return
 	}
 
 	return
 }
 
+// handleNotice processes NOTICE messages from the relay
+func (rc *RelayConnection) handleNotice(notice []byte) {
+	noticeStr := string(notice)
+	log.D.F("spider: NOTICE from %s: '%s'", rc.url, noticeStr)
+
+	// Check for rate limiting errors
+	if strings.Contains(noticeStr, "too many concurrent REQs") ||
+		strings.Contains(noticeStr, "rate limit") ||
+		strings.Contains(noticeStr, "slow down") {
+		rc.handleRateLimit()
+	}
+}
+
+// handleRateLimit applies backoff when rate limiting is detected
+func (rc *RelayConnection) handleRateLimit() {
+	rc.mu.Lock()
+	defer rc.mu.Unlock()
+
+	// Initialize backoff if not set
+	if rc.rateLimitBackoff == 0 {
+		rc.rateLimitBackoff = RateLimitBackoffDuration
+	} else {
+		// Exponential backoff
+		rc.rateLimitBackoff *= RateLimitBackoffMultiplier
+		if rc.rateLimitBackoff > MaxRateLimitBackoff {
+			rc.rateLimitBackoff = MaxRateLimitBackoff
+		}
+	}
+
+	rc.rateLimitUntil = time.Now().Add(rc.rateLimitBackoff)
+	log.W.F("spider: rate limit detected on %s, backing off for %v until %v",
+		rc.url, rc.rateLimitBackoff, rc.rateLimitUntil)
+
+	// Close all current subscriptions to reduce load
+	rc.clearSubscriptionsLocked()
+}
+
 // waitBeforeReconnect waits before attempting to reconnect with exponential backoff
 func (rc *RelayConnection) waitBeforeReconnect() {
+	log.I.F("spider: waiting %v before reconnecting to %s", rc.reconnectDelay, rc.url)
+
 	select {
 	case <-rc.ctx.Done():
 		return
@@ -342,12 +444,14 @@ func (rc *RelayConnection) waitBeforeReconnect() {
 	}
 
 	// Exponential backoff - double every time
+	// 10s -> 20s -> 40s -> 80s (1.3m) -> 160s (2.7m) -> 320s (5.3m) -> 640s (10.7m) -> 1280s (21m) -> 2560s (42m) -> 3600s (1h)
 	rc.reconnectDelay *= 2
-	
-	// If backoff exceeds 5 minutes, blackout for 24 hours
+
+	// Cap at MaxReconnectDelay (1 hour), then switch to 24-hour blackout
 	if rc.reconnectDelay >= MaxReconnectDelay {
 		rc.blackoutUntil = time.Now().Add(BlackoutPeriod)
-		log.W.F("spider: max backoff exceeded for %s (reached %v), blacking out for 24 hours", rc.url, rc.reconnectDelay)
+		rc.reconnectDelay = ReconnectDelay // Reset for after blackout
+		log.W.F("spider: max reconnect backoff reached for %s, entering 24-hour blackout period", rc.url)
 	}
 }
 
@@ -375,7 +479,24 @@ func (rc *RelayConnection) handleDisconnection() {
 // createSubscriptions creates batch subscriptions for the follow list
 func (rc *RelayConnection) createSubscriptions(followList [][]byte) {
 	rc.mu.Lock()
-	defer rc.mu.Unlock()
+
+	// Check if we're in a rate limit backoff period
+	if time.Now().Before(rc.rateLimitUntil) {
+		remaining := time.Until(rc.rateLimitUntil)
+		rc.mu.Unlock()
+		log.W.F("spider: skipping subscription creation for %s, rate limited for %v more", rc.url, remaining)
+
+		// Schedule retry after backoff period
+		go func() {
+			time.Sleep(remaining)
+			rc.createSubscriptions(followList)
+		}()
+		return
+	}
+
+	// Clear rate limit backoff on successful subscription attempt
+	rc.rateLimitBackoff = 0
+	rc.rateLimitUntil = time.Time{}
 
 	// Clear existing subscriptions
 	rc.clearSubscriptionsLocked()
@@ -386,9 +507,27 @@ func (rc *RelayConnection) createSubscriptions(followList [][]byte) {
 	log.I.F("spider: creating %d subscription batches for %d pubkeys on %s",
 		len(batches), len(followList), rc.url)
 
+	// Release lock before creating subscriptions to avoid holding it during delays
+	rc.mu.Unlock()
+
 	for i, batch := range batches {
-		batchID := fmt.Sprintf("batch-%d", i) // Simple batch ID
+		// Check context before creating each batch
+		select {
+		case <-rc.ctx.Done():
+			return
+		default:
+		}
+
+		batchID := fmt.Sprintf("batch-%d", i)
+
+		rc.mu.Lock()
 		rc.createBatchSubscription(batchID, batch)
+		rc.mu.Unlock()
+
+		// Add delay between batches to avoid overwhelming the relay
+		if i < len(batches)-1 { // Don't delay after the last batch
+			time.Sleep(BatchCreationDelay)
+		}
 	}
 }
 
@@ -457,6 +596,10 @@ func (rc *RelayConnection) createBatchSubscription(batchID string, pubkeys [][]b
 
 // handleEvents processes events from the subscription
 func (bs *BatchSubscription) handleEvents() {
+	// Throttle event processing to avoid CPU spikes
+	ticker := time.NewTicker(10 * time.Millisecond)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-bs.relay.ctx.Done():
@@ -466,13 +609,19 @@ func (bs *BatchSubscription) handleEvents() {
 				return // Subscription closed
 			}
 
+			// Wait for throttle tick to avoid processing events too rapidly
+			<-ticker.C
+
 			// Save event to database
 			if _, err := bs.relay.spider.db.SaveEvent(bs.relay.ctx, ev); err != nil {
+				// Ignore duplicate events and other errors
+				log.T.F("spider: failed to save event from %s: %v", bs.relay.url, err)
 			} else {
 				// Publish event if it was newly saved
 				if bs.relay.spider.pub != nil {
 					go bs.relay.spider.pub.Deliver(ev)
 				}
+				log.T.F("spider: saved event from %s", bs.relay.url)
 			}
 		}
 	}
@@ -485,7 +634,14 @@ func (rc *RelayConnection) updateSubscriptions(followList [][]byte) {
 	}
 
 	rc.mu.Lock()
-	defer rc.mu.Unlock()
+
+	// Check if we're in a rate limit backoff period
+	if time.Now().Before(rc.rateLimitUntil) {
+		remaining := time.Until(rc.rateLimitUntil)
+		rc.mu.Unlock()
+		log.D.F("spider: deferring subscription update for %s, rate limited for %v more", rc.url, remaining)
+		return
+	}
 
 	// Check if we need to perform catch-up for disconnected subscriptions
 	now := time.Now()
@@ -507,9 +663,28 @@ func (rc *RelayConnection) updateSubscriptions(followList [][]byte) {
 	rc.clearSubscriptionsLocked()
 
 	batches := rc.createBatches(followList)
+
+	// Release lock before creating subscriptions
+	rc.mu.Unlock()
+
 	for i, batch := range batches {
+		// Check context before creating each batch
+		select {
+		case <-rc.ctx.Done():
+			return
+		default:
+		}
+
 		batchID := fmt.Sprintf("batch-%d", i)
+
+		rc.mu.Lock()
 		rc.createBatchSubscription(batchID, batch)
+		rc.mu.Unlock()
+
+		// Add delay between batches
+		if i < len(batches)-1 {
+			time.Sleep(BatchCreationDelay)
+		}
 	}
 }
 
@@ -559,39 +734,43 @@ func (rc *RelayConnection) performCatchup(sub *BatchSubscription, disconnectTime
 	}
 	defer catchupSub.Unsub()
 
-	// Process catch-up events
+	// Process catch-up events with throttling
 	eventCount := 0
-	timeout := time.After(30 * time.Second)
+	timeout := time.After(60 * time.Second) // Increased timeout for catch-up
+	throttle := time.NewTicker(20 * time.Millisecond)
+	defer throttle.Stop()
 
 	for {
 		select {
 		case <-catchupCtx.Done():
-			log.D.F("spider: catch-up completed on %s, processed %d events", rc.url, eventCount)
+			log.I.F("spider: catch-up completed on %s, processed %d events", rc.url, eventCount)
 			return
 		case <-timeout:
-			log.D.F("spider: catch-up timeout on %s, processed %d events", rc.url, eventCount)
+			log.I.F("spider: catch-up timeout on %s, processed %d events", rc.url, eventCount)
 			return
 		case <-catchupSub.EndOfStoredEvents:
-			log.D.F("spider: catch-up EOSE on %s, processed %d events", rc.url, eventCount)
+			log.I.F("spider: catch-up EOSE on %s, processed %d events", rc.url, eventCount)
 			return
 		case ev := <-catchupSub.Events:
 			if ev == nil {
 				return
 			}
 
+			// Throttle event processing
+			<-throttle.C
+
 			eventCount++
 
 			// Save event to database
 			if _, err := rc.spider.db.SaveEvent(rc.ctx, ev); err != nil {
-				if !chk.E(err) {
-					log.T.F("spider: catch-up saved event %s from %s",
-						hex.Enc(ev.ID[:]), rc.url)
-				}
+				// Silently ignore errors (mostly duplicates)
 			} else {
 				// Publish event if it was newly saved
 				if rc.spider.pub != nil {
 					go rc.spider.pub.Deliver(ev)
 				}
+				log.T.F("spider: catch-up saved event %s from %s",
+					hex.Enc(ev.ID[:]), rc.url)
 			}
 		}
 	}

pkg/version/version

@@ -1 +1 @@
-v0.28.1
+v0.29.0

pkg/wasm/.claude/settings.local.json

@@ -0,0 +1,10 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(go build:*)",
+      "Bash(CGO_ENABLED=0 go build:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

pkg/wasm/hello/README.md

@@ -0,0 +1,102 @@
+# WebAssembly Test Server
+
+Simple Go web server for serving WebAssembly files with correct MIME types.
+
+## Quick Start
+
+```bash
+# Build and run the server
+go run server.go
+
+# Or with custom port
+go run server.go -port 3000
+
+# Or serve from a different directory
+go run server.go -dir /path/to/wasm/files
+```
+
+## Build and Install
+
+```bash
+# Build binary
+go build -o wasm-server server.go
+
+# Run
+./wasm-server
+
+# Install to PATH
+go install
+```
+
+## Usage
+
+Once the server is running, open your browser to:
+- http://localhost:8080/
+
+The server will serve:
+- `index.html` - Main HTML page
+- `hello.js` - JavaScript loader for WASM
+- `hello.wasm` - WebAssembly binary module
+- `hello.wat` - WebAssembly text format (for reference)
+
+## Files
+
+- **server.go** - Go web server with WASM MIME type support
+- **index.html** - HTML page that loads the WASM module
+- **hello.js** - JavaScript glue code to instantiate and run WASM
+- **hello.wasm** - Compiled WebAssembly binary
+- **hello.wat** - WebAssembly text format source
+
+## Building WASM Files
+
+### From WAT (WebAssembly Text Format)
+
+```bash
+# Install wabt tools
+sudo apt install wabt
+
+# Compile WAT to WASM
+wat2wasm hello.wat -o hello.wasm
+
+# Disassemble WASM back to WAT
+wasm2wat hello.wasm -o hello.wat
+```
+
+### From Go (using TinyGo)
+
+```bash
+# Install TinyGo
+wget https://github.com/tinygo-org/tinygo/releases/download/v0.31.0/tinygo_0.31.0_amd64.deb
+sudo dpkg -i tinygo_0.31.0_amd64.deb
+
+# Create Go program
+cat > main.go << 'EOF'
+package main
+
+import "fmt"
+
+func main() {
+    fmt.Println("Hello from Go WASM!")
+}
+EOF
+
+# Compile to WASM
+tinygo build -o main.wasm -target=wasm main.go
+
+# Get the WASM runtime helper
+cp $(tinygo env TINYGOROOT)/targets/wasm_exec.js .
+```
+
+## Browser Console
+
+Open your browser's developer console (F12) to see the output from the WASM module.
+
+The `hello.wasm` module should print "Hello, World!" to the console.
+
+## CORS Headers
+
+The server includes CORS headers to allow:
+- Cross-origin requests during development
+- Loading WASM modules from different origins
+
+This is useful when developing and testing WASM modules.

pkg/wasm/hello/hello.js

@@ -0,0 +1,18 @@
+const memory = new WebAssembly.Memory({ initial: 1 });
+
+const log = (offset, length) => {
+  const bytes = new Uint8Array(memory.buffer, offset, length);
+  const string = new TextDecoder('utf8').decode(bytes);
+
+  console.log(string);
+};
+
+(async () => {
+  const response = await fetch('./hello.wasm');
+  const bytes = await response.arrayBuffer();
+  const { instance } = await WebAssembly.instantiate(bytes, {
+    env: { log, memory }
+  });
+
+  instance.exports.hello();
+})();

pkg/wasm/hello/index.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>Hello, World! in WebAssembly</title>
+  </head>
+  <body>
+    <script src="hello.js" type="module"></script>
+  </body>
+</html>

pkg/wasm/hello/server.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"path/filepath"
+)
+
+func main() {
+	port := flag.Int("port", 8080, "Port to serve on")
+	dir := flag.String("dir", ".", "Directory to serve files from")
+	flag.Parse()
+
+	// Create file server
+	fs := http.FileServer(http.Dir(*dir))
+
+	// Wrap with MIME type handler for WASM files
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		// Set correct MIME type for WebAssembly files
+		if filepath.Ext(r.URL.Path) == ".wasm" {
+			w.Header().Set("Content-Type", "application/wasm")
+		}
+
+		// Set CORS headers to allow cross-origin requests (useful for development)
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
+
+		// Handle OPTIONS preflight requests
+		if r.Method == "OPTIONS" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+
+		fs.ServeHTTP(w, r)
+	})
+
+	addr := fmt.Sprintf(":%d", *port)
+	log.Printf("Starting WASM server on http://localhost%s", addr)
+	log.Printf("Serving files from: %s", *dir)
+	log.Printf("\nOpen http://localhost%s/ in your browser", addr)
+
+	if err := http.ListenAndServe(addr, nil); err != nil {
+		log.Fatal(err)
+	}
+}

pkg/wasm/shell/QUICKSTART.md

@@ -0,0 +1,125 @@
+# Quick Start Guide
+
+## TL;DR
+
+```bash
+# Build all examples
+./build.sh
+
+# Run hello example (stdout only)
+./run.sh hello.wasm
+
+# Run echo example (stdin/stdout)
+echo "test" | ./run.sh echo.wasm
+
+# Run all tests
+./test.sh
+```
+
+## What's Included
+
+### Scripts
+- **`build.sh`** - Compile all `.wat` files to `.wasm` using `wat2wasm`
+- **`run.sh`** - Execute WASM files with `wasmtime` WASI runtime
+- **`test.sh`** - Run complete test suite
+
+### Examples
+- **`hello.wat/wasm`** - Simple "Hello World" to stdout
+- **`echo.wat/wasm`** - Read from stdin, echo to stdout
+
+### Documentation
+- **`README.md`** - Complete documentation with examples
+- **`QUICKSTART.md`** - This file
+
+## Running WASM in Shell - The Basics
+
+### Console Output (stdout)
+```bash
+./run.sh hello.wasm
+# Output: Hello from WASM shell!
+```
+
+### Console Input (stdin)
+```bash
+# Piped input
+echo "your text" | ./run.sh echo.wasm
+
+# Interactive input
+./run.sh echo.wasm
+# (type your input and press Enter)
+
+# From file
+cat file.txt | ./run.sh echo.wasm
+```
+
+## Use Case: ORLY Policy Scripts
+
+This WASM shell runner is perfect for ORLY's policy system:
+
+```bash
+# Event JSON comes via stdin
+echo '{"kind":1,"content":"hello","pubkey":"..."}' | ./run.sh policy.wasm
+
+# Policy script:
+# - Reads JSON from stdin
+# - Applies rules
+# - Outputs decision to stdout: "accept" or "reject"
+
+# ORLY reads the decision and acts accordingly
+```
+
+### Benefits
+- **Sandboxed** - Cannot access system unless explicitly granted
+- **Fast** - Near-native performance with wasmtime's JIT
+- **Portable** - Same WASM binary runs everywhere
+- **Multi-language** - Write policies in Go, Rust, C, JavaScript, etc.
+- **Deterministic** - Same input = same output, always
+
+## Next Steps
+
+1. **Read the full README** - `cat README.md`
+2. **Try the examples** - `./test.sh`
+3. **Write your own** - Start with the template in README.md
+4. **Compile from Go** - Use TinyGo to compile Go to WASM
+5. **Integrate with ORLY** - Use as policy execution engine
+
+## File Structure
+
+```
+pkg/wasm/shell/
+├── build.sh           # Build script (wat -> wasm)
+├── run.sh             # Run script (execute wasm)
+├── test.sh            # Test all examples
+├── hello.wat          # Source: Hello World
+├── hello.wasm         # Binary: Hello World
+├── echo.wat           # Source: Echo stdin/stdout
+├── echo.wasm          # Binary: Echo stdin/stdout
+├── README.md          # Full documentation
+└── QUICKSTART.md      # This file
+```
+
+## Troubleshooting
+
+### "wasmtime not found"
+```bash
+curl https://wasmtime.dev/install.sh -sSf | bash
+export PATH="$HOME/.wasmtime/bin:$PATH"
+```
+
+### "wat2wasm not found"
+```bash
+sudo apt install wabt
+```
+
+### WASM fails to run
+```bash
+# Rebuild from source
+./build.sh
+
+# Check the WASM module
+wasm-objdump -x your.wasm
+```
+
+---
+
+**Happy WASM hacking!** 🎉

pkg/wasm/shell/README.md

@@ -0,0 +1,353 @@
+# WASM Shell Runner
+
+Run WebAssembly programs directly in your shell with stdin/stdout support using WASI (WebAssembly System Interface).
+
+## Quick Start
+
+```bash
+# Build all WAT files to WASM
+./build.sh
+
+# Run the hello example
+./run.sh hello.wasm
+
+# Run the echo example (with stdin)
+echo "Hello World" | ./run.sh echo.wasm
+
+# Or interactive
+./run.sh echo.wasm
+```
+
+## Prerequisites
+
+### Install wabt (WebAssembly Binary Toolkit)
+
+```bash
+# Ubuntu/Debian
+sudo apt install wabt
+
+# Provides: wat2wasm, wasm2wat, wasm-objdump, etc.
+```
+
+### Install wasmtime (WASM Runtime)
+
+```bash
+# Install via official installer
+curl https://wasmtime.dev/install.sh -sSf | bash
+
+# Add to PATH (add to ~/.bashrc for persistence)
+export PATH="$HOME/.wasmtime/bin:$PATH"
+```
+
+## Examples
+
+### 1. Hello World (`hello.wat`)
+
+Simple example that prints to stdout:
+
+```bash
+./build.sh
+./run.sh hello.wasm
+```
+
+**Output:**
+```
+Hello from WASM shell!
+```
+
+### 2. Echo Program (`echo.wat`)
+
+Reads from stdin and echoes back:
+
+```bash
+./build.sh
+
+# Interactive mode
+./run.sh echo.wasm
+# Type something and press Enter
+
+# Piped input
+echo "Test message" | ./run.sh echo.wasm
+
+# From file
+cat somefile.txt | ./run.sh echo.wasm
+```
+
+**Output:**
+```
+Enter text: Test message
+You entered: Test message
+```
+
+## How It Works
+
+### WASI (WebAssembly System Interface)
+
+WASI provides a standard interface for WASM programs to interact with the host system:
+
+- **stdin** (fd 0) - Standard input
+- **stdout** (fd 1) - Standard output
+- **stderr** (fd 2) - Standard error
+
+### Key WASI Functions Used
+
+#### `fd_write` - Write to file descriptor
+```wat
+(import "wasi_snapshot_preview1" "fd_write"
+  (func $fd_write (param i32 i32 i32 i32) (result i32)))
+
+;; Usage: fd_write(fd, iovs_ptr, iovs_len, nwritten_ptr) -> errno
+;; fd: File descriptor (1 = stdout)
+;; iovs_ptr: Pointer to iovec array
+;; iovs_len: Number of iovecs
+;; nwritten_ptr: Where to store bytes written
+```
+
+#### `fd_read` - Read from file descriptor
+```wat
+(import "wasi_snapshot_preview1" "fd_read"
+  (func $fd_read (param i32 i32 i32 i32) (result i32)))
+
+;; Usage: fd_read(fd, iovs_ptr, iovs_len, nread_ptr) -> errno
+;; fd: File descriptor (0 = stdin)
+;; iovs_ptr: Pointer to iovec array
+;; iovs_len: Number of iovecs
+;; nread_ptr: Where to store bytes read
+```
+
+### iovec Structure
+
+Both functions use an iovec (I/O vector) structure:
+
+```
+struct iovec {
+    u32 buf;      // Pointer to buffer in WASM memory
+    u32 buf_len;  // Length of buffer
+}
+```
+
+## Writing Your Own WASM Programs
+
+### Basic Template
+
+```wat
+(module
+  ;; Import WASI functions you need
+  (import "wasi_snapshot_preview1" "fd_write"
+    (func $fd_write (param i32 i32 i32 i32) (result i32)))
+
+  ;; Allocate memory
+  (memory 1)
+  (export "memory" (memory 0))
+
+  ;; Store your strings in memory
+  (data (i32.const 0) "Your message here\n")
+
+  ;; Main function (entry point)
+  (func $main (export "_start")
+    ;; Setup iovec at some offset (e.g., 100)
+    (i32.store (i32.const 100) (i32.const 0))   ;; buf pointer
+    (i32.store (i32.const 104) (i32.const 18))  ;; buf length
+
+    ;; Write to stdout
+    (call $fd_write
+      (i32.const 1)   ;; stdout
+      (i32.const 100) ;; iovec pointer
+      (i32.const 1)   ;; number of iovecs
+      (i32.const 200) ;; nwritten pointer
+    )
+    drop ;; drop return value
+  )
+)
+```
+
+### Build and Run
+
+```bash
+# Compile WAT to WASM
+wat2wasm yourprogram.wat -o yourprogram.wasm
+
+# Run it
+./run.sh yourprogram.wasm
+
+# Or directly with wasmtime
+wasmtime yourprogram.wasm
+```
+
+## Advanced Usage
+
+### Pass Arguments
+
+```bash
+# WASM programs can receive command-line arguments
+./run.sh program.wasm arg1 arg2 arg3
+```
+
+### Environment Variables
+
+```bash
+# Set environment variables (wasmtime flag)
+wasmtime --env KEY=value program.wasm
+```
+
+### Mount Directories
+
+```bash
+# Give WASM access to directories (wasmtime flag)
+wasmtime --dir=/tmp program.wasm
+```
+
+### Call Specific Functions
+
+```bash
+# Instead of _start, call a specific exported function
+wasmtime --invoke my_function program.wasm
+```
+
+## Compiling from High-Level Languages
+
+### From Go (using TinyGo)
+
+```bash
+# Install TinyGo
+wget https://github.com/tinygo-org/tinygo/releases/download/v0.31.0/tinygo_0.31.0_amd64.deb
+sudo dpkg -i tinygo_0.31.0_amd64.deb
+
+# Write Go program
+cat > main.go << 'EOF'
+package main
+
+import "fmt"
+
+func main() {
+    fmt.Println("Hello from Go WASM!")
+}
+EOF
+
+# Compile to WASM with WASI
+tinygo build -o program.wasm -target=wasi main.go
+
+# Run
+./run.sh program.wasm
+```
+
+### From Rust
+
+```bash
+# Add WASI target
+rustup target add wasm32-wasi
+
+# Create project
+cargo new --bin myprogram
+cd myprogram
+
+# Build for WASI
+cargo build --target wasm32-wasi --release
+
+# Run
+wasmtime target/wasm32-wasi/release/myprogram.wasm
+```
+
+### From C/C++ (using wasi-sdk)
+
+```bash
+# Download wasi-sdk
+wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0-linux.tar.gz
+tar xf wasi-sdk-21.0-linux.tar.gz
+
+# Compile C program
+cat > hello.c << 'EOF'
+#include <stdio.h>
+
+int main() {
+    printf("Hello from C WASM!\n");
+    return 0;
+}
+EOF
+
+# Compile to WASM
+./wasi-sdk-21.0/bin/clang hello.c -o hello.wasm
+
+# Run
+wasmtime hello.wasm
+```
+
+## Debugging
+
+### Inspect WASM Module
+
+```bash
+# Disassemble WASM to WAT
+wasm2wat program.wasm -o program.wat
+
+# Show module structure
+wasm-objdump -x program.wasm
+
+# Show imports
+wasm-objdump -x program.wasm | grep -A 10 "Import"
+
+# Show exports
+wasm-objdump -x program.wasm | grep -A 10 "Export"
+```
+
+### Verbose Execution
+
+```bash
+# Run with logging
+WASMTIME_LOG=wasmtime=trace wasmtime program.wasm
+
+# Enable debug info
+wasmtime -g program.wasm
+```
+
+## Use Cases for ORLY
+
+WASM with WASI is perfect for ORLY's policy system:
+
+### Sandboxed Policy Scripts
+
+```bash
+# Write policy in any language that compiles to WASM
+# Run it safely in a sandbox with controlled stdin/stdout
+./run.sh policy.wasm < event.json
+```
+
+**Benefits:**
+- **Security**: Sandboxed execution, no system access unless granted
+- **Portability**: Same WASM runs on any platform
+- **Performance**: Near-native speed with wasmtime's JIT
+- **Language Choice**: Write policies in Go, Rust, C, JavaScript, etc.
+- **Deterministic**: Same input always produces same output
+
+### Example Policy Flow
+
+```bash
+# Event comes in via stdin (JSON)
+echo '{"kind":1,"content":"hello"}' | ./run.sh filter-policy.wasm
+
+# Policy outputs "accept" or "reject" to stdout
+# ORLY reads the decision and acts accordingly
+```
+
+## Scripts Reference
+
+### `build.sh`
+Compiles all `.wat` files to `.wasm` using `wat2wasm`
+
+### `run.sh [wasm-file] [args...]`
+Runs a WASM file with `wasmtime`, defaults to `hello.wasm`
+
+## Files
+
+- **hello.wat** - Simple stdout example
+- **echo.wat** - stdin/stdout interactive example
+- **build.sh** - Build all WAT files
+- **run.sh** - Run WASM files with wasmtime
+
+## Resources
+
+- [WASI Specification](https://github.com/WebAssembly/WASI)
+- [Wasmtime Documentation](https://docs.wasmtime.dev/)
+- [WebAssembly Reference](https://webassembly.github.io/spec/)
+- [WAT Language Guide](https://developer.mozilla.org/en-US/docs/WebAssembly/Understanding_the_text_format)
+- [TinyGo WASI Support](https://tinygo.org/docs/guides/webassembly/wasi/)

pkg/wasm/shell/build.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Build script for WASM shell examples
+# Compiles WAT (WebAssembly Text) to WASM binary
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+echo "Building WASM modules from WAT files..."
+
+# Check if wat2wasm is available
+if ! command -v wat2wasm &> /dev/null; then
+    echo "Error: wat2wasm not found. Install wabt:"
+    echo "  sudo apt install wabt"
+    exit 1
+fi
+
+# Build each .wat file to .wasm
+for wat_file in *.wat; do
+    if [ -f "$wat_file" ]; then
+        wasm_file="${wat_file%.wat}.wasm"
+        echo "  $wat_file -> $wasm_file"
+        wat2wasm "$wat_file" -o "$wasm_file"
+    fi
+done
+
+echo "Build complete!"
+echo ""
+echo "Run with:"
+echo "  ./run.sh hello.wasm"
+echo "  or"
+echo "  wasmtime hello.wasm"

pkg/wasm/shell/run.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Run script for WASM shell examples
+# Executes WASM files using wasmtime with WASI support
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Find wasmtime executable
+WASMTIME=""
+if command -v wasmtime &> /dev/null; then
+    WASMTIME="wasmtime"
+elif [ -x "$HOME/.wasmtime/bin/wasmtime" ]; then
+    WASMTIME="$HOME/.wasmtime/bin/wasmtime"
+else
+    echo "Error: wasmtime not found. Install it:"
+    echo "  curl https://wasmtime.dev/install.sh -sSf | bash"
+    echo ""
+    echo "Or add to PATH:"
+    echo "  export PATH=\"\$HOME/.wasmtime/bin:\$PATH\""
+    exit 1
+fi
+
+# Get the WASM file from argument, default to hello.wasm
+WASM_FILE="${1:-hello.wasm}"
+
+# If relative path, make it relative to script dir
+if [[ "$WASM_FILE" != /* ]]; then
+    WASM_FILE="$SCRIPT_DIR/$WASM_FILE"
+fi
+
+if [ ! -f "$WASM_FILE" ]; then
+    echo "Error: WASM file not found: $WASM_FILE"
+    echo ""
+    echo "Usage: $0 [wasm-file]"
+    echo ""
+    echo "Available WASM files:"
+    cd "$SCRIPT_DIR"
+    ls -1 *.wasm 2>/dev/null || echo "  (none - run ./build.sh first)"
+    exit 1
+fi
+
+echo "Running: $WASM_FILE"
+echo "---"
+
+# Run the WASM file with wasmtime
+# Additional flags you might want:
+#   --dir=. : Mount current directory
+#   --env VAR=value : Set environment variable
+#   --invoke function : Call specific function instead of _start
+"$WASMTIME" "$WASM_FILE" "$@"

pkg/wasm/shell/test.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Test script for WASM shell examples
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+echo "========================================="
+echo "WASM Shell Test Suite"
+echo "========================================="
+echo ""
+
+# Build first
+echo "[1/4] Building WASM modules..."
+./build.sh
+echo ""
+
+# Test hello.wasm
+echo "[2/4] Testing hello.wasm (stdout only)..."
+echo "---"
+./run.sh hello.wasm
+echo ""
+
+# Test echo.wasm with piped input
+echo "[3/4] Testing echo.wasm (stdin/stdout with pipe)..."
+echo "---"
+echo "This is a test message" | ./run.sh echo.wasm
+echo ""
+
+# Test echo.wasm with heredoc
+echo "[4/4] Testing echo.wasm (stdin/stdout with heredoc)..."
+echo "---"
+./run.sh echo.wasm <<< "Testing heredoc input"
+echo ""
+
+echo "========================================="
+echo "All tests passed!"
+echo "========================================="
+echo ""
+echo "Try these commands:"
+echo "  ./run.sh hello.wasm"
+echo "  echo 'your text' | ./run.sh echo.wasm"
+echo "  ./run.sh echo.wasm  # interactive mode"