Implement spider functionality for event synchronization

- Introduced a new `spider` package to manage connections to admin relays and synchronize events for followed pubkeys.
- Added configuration options for spider mode in the application settings, allowing for different operational modes (e.g., follows).
- Implemented callback mechanisms to dynamically retrieve admin relays and follow lists.
- Enhanced the main application to initialize and manage the spider, including starting and stopping its operation.
- Added tests to validate spider creation, callbacks, and operational behavior.
- Bumped version to v0.17.14.

app/config/config.go

@@ -59,6 +59,9 @@ type C struct {
 	// Sprocket settings
 	SprocketEnabled bool `env:"ORLY_SPROCKET_ENABLED" default:"false" usage:"enable sprocket event processing plugin system"`
 
+	// Spider settings
+	SpiderMode string `env:"ORLY_SPIDER_MODE" default:"none" usage:"spider mode for syncing events: none, follows"`
+
 	PolicyEnabled bool `env:"ORLY_POLICY_ENABLED" default:"false" usage:"enable policy-based event processing (configuration found in $HOME/.config/ORLY/policy.json)"`
 }
 

app/handle-message.go

@@ -3,6 +3,7 @@ package app
 import (
 	"fmt"
 	"time"
+	"unicode"
 
 	"lol.mleku.dev/chk"
 	"lol.mleku.dev/log"
@@ -15,6 +16,42 @@ import (
 	"next.orly.dev/pkg/encoders/envelopes/reqenvelope"
 )
 
+// validateJSONMessage checks if a message contains invalid control characters
+// that would cause JSON parsing to fail
+func validateJSONMessage(msg []byte) (err error) {
+	for i, b := range msg {
+		// Check for invalid control characters in JSON strings
+		if b < 32 && b != '\t' && b != '\n' && b != '\r' {
+			// Allow some control characters that might be valid in certain contexts
+			// but reject form feed (\f), backspace (\b), and other problematic ones
+			switch b {
+			case '\b', '\f', 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+				0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+				0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F:
+				return fmt.Errorf("invalid control character 0x%02X at position %d", b, i)
+			}
+		}
+		// Check for non-printable characters that might indicate binary data
+		if b > 127 && !unicode.IsPrint(rune(b)) {
+			// Allow valid UTF-8 sequences, but be suspicious of random binary data
+			if i < len(msg)-1 {
+				// Quick check: if we see a lot of high-bit characters in sequence,
+				// it might be binary data masquerading as text
+				highBitCount := 0
+				for j := i; j < len(msg) && j < i+10; j++ {
+					if msg[j] > 127 {
+						highBitCount++
+					}
+				}
+				if highBitCount > 7 { // More than 70% high-bit chars in a 10-byte window
+					return fmt.Errorf("suspicious binary data detected at position %d", i)
+				}
+			}
+		}
+	}
+	return
+}
+
 func (l *Listener) HandleMessage(msg []byte, remote string) {
 	// Handle blacklisted IPs - discard messages but keep connection open until timeout
 	if l.isBlacklisted {
@@ -35,6 +72,17 @@ func (l *Listener) HandleMessage(msg []byte, remote string) {
 	}
 	// log.D.F("%s processing message (len=%d): %s", remote, len(msg), msgPreview)
 
+	// Validate message for invalid characters before processing
+	if err := validateJSONMessage(msg); err != nil {
+		log.E.F("%s message validation FAILED (len=%d): %v", remote, len(msg), err)
+		log.T.F("%s invalid message content: %q", remote, msgPreview)
+		// Send error notice to client
+		if noticeErr := noticeenvelope.NewFrom("invalid message format: " + err.Error()).Write(l); noticeErr != nil {
+			log.E.F("%s failed to send validation error notice: %v", remote, noticeErr)
+		}
+		return
+	}
+
 	l.msgCount++
 	var err error
 	var t string

app/handle-req.go

@@ -35,6 +35,12 @@ func (l *Listener) HandleReq(msg []byte) (err error) {
 	// var rem []byte
 	env := reqenvelope.New()
 	if _, err = env.Unmarshal(msg); chk.E(err) {
+		// Provide more specific error context for JSON parsing failures
+		if strings.Contains(err.Error(), "invalid character") {
+			log.E.F("REQ JSON parsing failed from %s: %v", l.remote, err)
+			log.T.F("REQ malformed message from %s: %q", l.remote, string(msg))
+			return normalize.Error.Errorf("malformed REQ message: %s", err.Error())
+		}
 		return normalize.Error.Errorf(err.Error())
 	}
 

app/main.go

@@ -10,11 +10,13 @@ import (
 	"lol.mleku.dev/chk"
 	"lol.mleku.dev/log"
 	"next.orly.dev/app/config"
+	"next.orly.dev/pkg/acl"
 	"next.orly.dev/pkg/crypto/keys"
 	"next.orly.dev/pkg/database"
 	"next.orly.dev/pkg/encoders/bech32encoding"
 	"next.orly.dev/pkg/policy"
 	"next.orly.dev/pkg/protocol/publish"
+	"next.orly.dev/pkg/spider"
 )
 
 func Run(
@@ -69,6 +71,48 @@ func Run(
 
 	// Initialize policy manager
 	l.policyManager = policy.NewWithManager(ctx, cfg.AppName, cfg.PolicyEnabled)
+
+	// Initialize spider manager based on mode
+	if cfg.SpiderMode != "none" {
+		if l.spiderManager, err = spider.New(ctx, db, l.publishers, cfg.SpiderMode); chk.E(err) {
+			log.E.F("failed to create spider manager: %v", err)
+		} else {
+			// Set up callbacks for follows mode
+			if cfg.SpiderMode == "follows" {
+				l.spiderManager.SetCallbacks(
+					func() []string {
+						// Get admin relays from follows ACL if available
+						for _, aclInstance := range acl.Registry.ACL {
+							if aclInstance.Type() == "follows" {
+								if follows, ok := aclInstance.(*acl.Follows); ok {
+									return follows.AdminRelays()
+								}
+							}
+						}
+						return nil
+					},
+					func() [][]byte {
+						// Get followed pubkeys from follows ACL if available
+						for _, aclInstance := range acl.Registry.ACL {
+							if aclInstance.Type() == "follows" {
+								if follows, ok := aclInstance.(*acl.Follows); ok {
+									return follows.GetFollowedPubkeys()
+								}
+							}
+						}
+						return nil
+					},
+				)
+			}
+
+			if err = l.spiderManager.Start(); chk.E(err) {
+				log.E.F("failed to start spider manager: %v", err)
+			} else {
+				log.I.F("spider manager started successfully in '%s' mode", cfg.SpiderMode)
+			}
+		}
+	}
+
 	// Initialize the user interface
 	l.UserInterface()
 
@@ -135,6 +179,12 @@ func Run(
 		<-ctx.Done()
 		log.I.F("shutting down HTTP server gracefully")
 
+		// Stop spider manager if running
+		if l.spiderManager != nil {
+			l.spiderManager.Stop()
+			log.I.F("spider manager stopped")
+		}
+
 		// Create shutdown context with timeout
 		shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancelShutdown()

app/server.go

@@ -26,6 +26,7 @@ import (
 	"next.orly.dev/pkg/protocol/auth"
 	"next.orly.dev/pkg/protocol/httpauth"
 	"next.orly.dev/pkg/protocol/publish"
+	"next.orly.dev/pkg/spider"
 )
 
 type Server struct {
@@ -47,6 +48,7 @@ type Server struct {
 	paymentProcessor *PaymentProcessor
 	sprocketManager  *SprocketManager
 	policyManager    *policy.P
+	spiderManager    *spider.Spider
 }
 
 // isIPBlacklisted checks if an IP address is blacklisted using the managed ACL system