lmdb: do not use goroutines for each query.

thanks to @wojas at https://github.com/PowerDNS/lmdb-go/issues/28#issuecomment-1845056613

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.25.7
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 )
 
 require (

go.sum

@@ -179,6 +179,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

lmdb/query.go

@@ -7,7 +7,7 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
-	"sync"
+	"log"
 
 	"github.com/PowerDNS/lmdb-go/lmdb"
 	"github.com/nbd-wtf/go-nostr"
@@ -29,31 +29,25 @@ type queryEvent struct {
 }
 
 func (b *LMDBBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
-	ch := make(chan *nostr.Event)
-
 	queries, extraFilter, since, err := b.prepareQueries(filter)
 	if err != nil {
 		return nil, err
 	}
 
+	ch := make(chan *nostr.Event)
 	go func() {
-		err := b.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		defer close(ch)
-			txn.RawRead = true
-			wg := sync.WaitGroup{}
-			wg.Add(len(queries))
 
-			// actually iterate
+		err := b.lmdbEnv.View(func(txn *lmdb.Txn) error {
-			cursorClosers := make([]func(), len(queries))
+			for _, q := range queries {
-			for i, q := range queries {
+				txn.RawRead = true
-				go func(i int, q query) {
 				defer close(q.results)
-					defer wg.Done()
 
 				cursor, err := txn.OpenCursor(q.dbi)
 				if err != nil {
-						return
+					return err
 				}
-					cursorClosers[i] = cursor.Close
+				defer cursor.Close()
 
 				var k []byte
 				var idx []byte
@@ -82,7 +76,8 @@ func (b *LMDBBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (cha
 
 					// we already have a k and a v and an err from the cursor setup, so check and use these
 					if iterr != nil || !bytes.HasPrefix(k, q.prefix) {
-							return
+						// either iteration has errored or we reached the end of this prefix
+						break // stop this cursor and move to the next one
 					}
 
 					// "id" indexes don't contain a timestamp
@@ -96,12 +91,16 @@ func (b *LMDBBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (cha
 					// fetch actual event
 					val, err := txn.Get(b.rawEventStore, idx)
 					if err != nil {
-							panic(err)
+						log.Printf(
+							"lmdb: failed to get %x based on prefix %x, index key %x from raw event store: %s\n",
+							idx, q.prefix, k, err)
+						continue
 					}
 
 					evt := &nostr.Event{}
 					if err := nostr_binary.Unmarshal(val, evt); err != nil {
-							panic(err)
+						log.Printf("lmdb: value read error: %s\n", err)
+						continue
 					}
 
 					// check if this matches the other filters that were not part of the index
@@ -112,7 +111,11 @@ func (b *LMDBBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (cha
 					// move one back (we'll look into k and v and err in the next iteration)
 					k, idx, iterr = cursor.Get(nil, nil, lmdb.Prev)
 				}
-				}(i, q)
+			}
+			return nil
+		})
+		if err != nil {
+			log.Printf("lmdb: error on cursor iteration: %v\n", err)
 		}
 
 		// max number of events we'll return
@@ -133,17 +136,9 @@ func (b *LMDBBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (cha
 			}
 		}
 
-			// now it's a good time to schedule this
-			defer func() {
-				close(ch)
-				for _, cclose := range cursorClosers {
-					cclose()
-				}
-			}()
-
 		// queue may be empty here if we have literally nothing
 		if len(emitQueue) == 0 {
-				return nil
+			return
 		}
 
 		heap.Init(&emitQueue)
@@ -174,13 +169,6 @@ func (b *LMDBBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (cha
 				}
 			}
 		}
-
-			wg.Wait()
-			return nil
-		})
-		if err != nil {
-			panic(err)
-		}
 	}()
 
 	return ch, nil