Implement distributed synchronization features

- Added a sync manager to handle distributed synchronization across relay peers, initialized in the main application run function.
- Enhanced the event handling to update the serial number for synchronization when events are processed.
- Introduced new API endpoints for synchronization, allowing peers to fetch the current serial number and events within a specified range.
- Implemented peer request validation for synchronization endpoints to ensure authorized access based on NIP-98 authentication.
- Updated configuration to support relay peers for synchronization.
- Bumped version to v0.24.0 to reflect these changes.

pkg/database/import.go

@@ -1,86 +1,17 @@
 package database
 
 import (
-	"bufio"
 	"io"
-	"os"
-	"runtime/debug"
 
 	"lol.mleku.dev/chk"
 	"lol.mleku.dev/log"
-	"next.orly.dev/pkg/encoders/event"
 )
 
-const maxLen = 500000000
-
 // Import a collection of events in line structured minified JSON format (JSONL).
 func (d *D) Import(rr io.Reader) {
-	// store to disk so we can return fast
-	tmpPath := os.TempDir() + string(os.PathSeparator) + "orly"
-	os.MkdirAll(tmpPath, 0700)
-	tmp, err := os.CreateTemp(tmpPath, "")
-	if chk.E(err) {
-		return
-	}
-	log.I.F("buffering upload to %s", tmp.Name())
-	if _, err = io.Copy(tmp, rr); chk.E(err) {
-		return
-	}
-	if _, err = tmp.Seek(0, 0); chk.E(err) {
-		return
-	}
-
 	go func() {
-		var err error
-		// Create a scanner to read the buffer line by line
-		scan := bufio.NewScanner(tmp)
-		scanBuf := make([]byte, maxLen)
-		scan.Buffer(scanBuf, maxLen)
-
-		var count, total int
-		for scan.Scan() {
-			select {
-			case <-d.ctx.Done():
-				log.I.F("context closed")
-				return
-			default:
-			}
-
-			b := scan.Bytes()
-			total += len(b) + 1
-			if len(b) < 1 {
-				continue
-			}
-
-			ev := event.New()
-			if _, err = ev.Unmarshal(b); err != nil {
-				// return the pooled buffer on error
-				ev.Free()
-				continue
-			}
-
-			if _, err = d.SaveEvent(d.ctx, ev); err != nil {
-				// return the pooled buffer on error paths too
-				ev.Free()
-				continue
-			}
-
-			// return the pooled buffer after successful save
-			ev.Free()
-			b = nil
-			count++
-			if count%100 == 0 {
-				log.I.F("received %d events", count)
-				debug.FreeOSMemory()
-			}
+		if err := d.ImportEventsFromReader(d.ctx, rr); chk.E(err) {
+			log.E.F("import failed: %v", err)
 		}
-
-		log.I.F("read %d bytes and saved %d events", total, count)
-		err = scan.Err()
-		if chk.E(err) {
-		}
-
-		// Help garbage collection
-		tmp = nil
 	}()
 }