Fix directory spider tag loss: size limits and validation
Some checks failed
Go / build-and-release (push) Has been cancelled
Some checks failed
Go / build-and-release (push) Has been cancelled
- Increase WebSocket message size limit from 500KB to 10MB to prevent truncation of large kind 3 follow list events (8000+ follows) - Add validation in SaveEvent to reject kind 3 events without p tags before storage, preventing malformed events from buggy relays - Implement CleanupKind3WithoutPTags() to remove existing malformed kind 3 events at startup - Add enhanced logging showing tag count and event ID when rejecting invalid kind 3 events for better observability - Create round-trip test proving binary tag encoding preserves p tags correctly through JSON→binary→JSON cycle - Root cause: 500KB limit was truncating large follow lists during WebSocket receive, causing tags to be lost or incomplete - Three-layer defense: prevent at gate (size), validate (save time), and cleanup (startup) Files modified: - app/handle-websocket.go: Increase DefaultMaxMessageSize to 10MB - pkg/database/save-event.go: Add kind 3 validation with logging - pkg/database/cleanup-kind3.go: New cleanup function - pkg/database/cleanup-kind3_test.go: Round-trip test - app/main.go: Invoke cleanup at startup
This commit is contained in:
72
pkg/database/cleanup-kind3.go
Normal file
72
pkg/database/cleanup-kind3.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.mleku.dev/mleku/nostr/encoders/filter"
|
||||
"git.mleku.dev/mleku/nostr/encoders/kind"
|
||||
"lol.mleku.dev/chk"
|
||||
"lol.mleku.dev/log"
|
||||
)
|
||||
|
||||
// CleanupKind3WithoutPTags scans for kind 3 follow list events that have no p tags
|
||||
// and deletes them. This cleanup is needed because the directory spider may have
|
||||
// saved malformed events that lost their tags during serialization.
|
||||
func (d *D) CleanupKind3WithoutPTags(ctx context.Context) error {
|
||||
log.I.F("database: starting cleanup of kind 3 events without p tags")
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
// Query for all kind 3 events
|
||||
f := &filter.F{
|
||||
Kinds: kind.NewS(kind.FollowList),
|
||||
}
|
||||
|
||||
events, err := d.QueryEvents(ctx, f)
|
||||
if chk.E(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
deletedCount := 0
|
||||
|
||||
// Check each event for p tags
|
||||
for _, ev := range events {
|
||||
hasPTag := false
|
||||
|
||||
if ev.Tags != nil && ev.Tags.Len() > 0 {
|
||||
// Look for at least one p tag
|
||||
for _, tag := range *ev.Tags {
|
||||
if tag != nil && tag.Len() >= 2 {
|
||||
key := tag.Key()
|
||||
if len(key) == 1 && key[0] == 'p' {
|
||||
hasPTag = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Delete events without p tags
|
||||
if !hasPTag {
|
||||
log.W.F("database: deleting kind 3 event without p tags from pubkey %x", ev.Pubkey)
|
||||
if err := d.DeleteEvent(ctx, ev.ID); chk.E(err) {
|
||||
log.E.F("database: failed to delete kind 3 event %x: %v", ev.ID, err)
|
||||
continue
|
||||
}
|
||||
deletedCount++
|
||||
}
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
|
||||
if deletedCount > 0 {
|
||||
log.I.F("database: cleanup completed in %v - deleted %d kind 3 events without p tags (scanned %d total)",
|
||||
duration, deletedCount, len(events))
|
||||
} else {
|
||||
log.I.F("database: cleanup completed in %v - no kind 3 events needed deletion (scanned %d total)",
|
||||
duration, len(events))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user