badger: use special index for 'a'-like tags.

badger/helpers.go

@@ -0,0 +1,125 @@
+package badger
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"strconv"
+	"strings"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func getAddrTagElements(tagValue string) (kind uint16, pkb []byte, d string) {
+	spl := strings.Split(tagValue, ":")
+	if len(spl) == 3 {
+		if pkb, _ := hex.DecodeString(spl[1]); len(pkb) == 32 {
+			if kind, err := strconv.ParseUint(spl[0], 10, 16); err == nil {
+				return uint16(kind), pkb, spl[2]
+			}
+		}
+	}
+	return 0, nil, ""
+}
+
+func getTagIndexPrefix(tagValue string) ([]byte, int) {
+	var k []byte   // the key with full length for created_at and idx at the end, but not filled with these
+	var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start
+
+	if kind, pkb, d := getAddrTagElements(tagValue); len(pkb) == 32 {
+		// store value in the new special "a" tag index
+		k = make([]byte, 1+2+32+len(d)+4+4)
+		k[0] = indexTagAddrPrefix
+		binary.BigEndian.PutUint16(k[1:], kind)
+		copy(k[1+2:], pkb)
+		copy(k[1+2+32:], d)
+		offset = 1 + 2 + 32 + len(d)
+	} else if vb, _ := hex.DecodeString(tagValue); len(vb) == 32 {
+		// store value as bytes
+		k = make([]byte, 1+32+4+4)
+		k[0] = indexTag32Prefix
+		copy(k[1:], vb)
+		offset = 1 + 32
+	} else {
+		// store whatever as utf-8
+		k = make([]byte, 1+len(tagValue)+4+4)
+		k[0] = indexTagPrefix
+		copy(k[1:], tagValue)
+		offset = 1 + len(tagValue)
+	}
+
+	return k, offset
+}
+
+func getIndexKeysForEvent(evt *nostr.Event, idx []byte) [][]byte {
+	keys := make([][]byte, 0, 18)
+
+	// indexes
+	{
+		// ~ by id
+		id, _ := hex.DecodeString(evt.ID)
+		k := make([]byte, 1+32+4)
+		k[0] = indexIdPrefix
+		copy(k[1:], id)
+		copy(k[1+32:], idx)
+		keys = append(keys, k)
+	}
+
+	{
+		// ~ by pubkey+date
+		pubkey, _ := hex.DecodeString(evt.PubKey)
+		k := make([]byte, 1+32+4+4)
+		k[0] = indexPubkeyPrefix
+		copy(k[1:], pubkey)
+		binary.BigEndian.PutUint32(k[1+32:], uint32(evt.CreatedAt))
+		copy(k[1+32+4:], idx)
+		keys = append(keys, k)
+	}
+
+	{
+		// ~ by kind+date
+		k := make([]byte, 1+2+4+4)
+		k[0] = indexKindPrefix
+		binary.BigEndian.PutUint16(k[1:], uint16(evt.Kind))
+		binary.BigEndian.PutUint32(k[1+2:], uint32(evt.CreatedAt))
+		copy(k[1+2+4:], idx)
+		keys = append(keys, k)
+	}
+
+	{
+		// ~ by pubkey+kind+date
+		pubkey, _ := hex.DecodeString(evt.PubKey)
+		k := make([]byte, 1+32+2+4+4)
+		k[0] = indexPubkeyKindPrefix
+		copy(k[1:], pubkey)
+		binary.BigEndian.PutUint16(k[1+32:], uint16(evt.Kind))
+		binary.BigEndian.PutUint32(k[1+32+2:], uint32(evt.CreatedAt))
+		copy(k[1+32+2+4:], idx)
+		keys = append(keys, k)
+	}
+
+	// ~ by tagvalue+date
+	for _, tag := range evt.Tags {
+		if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 {
+			continue
+		}
+
+		// get key prefix (with full length) and offset where to write the last parts
+		k, offset := getTagIndexPrefix(tag[1])
+
+		// write the last parts (created_at and idx)
+		binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt))
+		copy(k[offset+4:], idx)
+		keys = append(keys, k)
+	}
+
+	{
+		// ~ by date only
+		k := make([]byte, 1+4+4)
+		k[0] = indexCreatedAtPrefix
+		binary.BigEndian.PutUint32(k[1:], uint32(evt.CreatedAt))
+		copy(k[1+4:], idx)
+		keys = append(keys, k)
+	}
+
+	return keys
+}

badger/lib.go

@@ -2,11 +2,10 @@ package badger
 
 import (
 	"encoding/binary"
-	"encoding/hex"
+	"fmt"
 
 	"github.com/dgraph-io/badger/v4"
 	"github.com/fiatjaf/eventstore"
-	"github.com/nbd-wtf/go-nostr"
 )
 
 const (
@@ -19,6 +18,7 @@ const (
 	indexPubkeyKindPrefix byte = 5
 	indexTagPrefix        byte = 6
 	indexTag32Prefix      byte = 7
+	indexTagAddrPrefix    byte = 8
 )
 
 var _ eventstore.Store = (*BadgerBackend)(nil)
@@ -42,6 +42,10 @@ func (b *BadgerBackend) Init() error {
 		return err
 	}
 
+	if err := b.runMigrations(); err != nil {
+		return fmt.Errorf("error running migrations: %w", err)
+	}
+
 	if b.MaxLimit == 0 {
 		b.MaxLimit = 500
 	}
@@ -61,87 +65,3 @@ func (b BadgerBackend) Serial() []byte {
 	binary.BigEndian.PutUint32(vb[1:], uint32(v))
 	return vb
 }
-
-func getIndexKeysForEvent(evt *nostr.Event, idx []byte) [][]byte {
-	keys := make([][]byte, 0, 18)
-
-	// indexes
-	{
-		// ~ by id
-		id, _ := hex.DecodeString(evt.ID)
-		k := make([]byte, 1+32+4)
-		k[0] = indexIdPrefix
-		copy(k[1:], id)
-		copy(k[1+32:], idx)
-		keys = append(keys, k)
-	}
-
-	{
-		// ~ by pubkey+date
-		pubkey, _ := hex.DecodeString(evt.PubKey)
-		k := make([]byte, 1+32+4+4)
-		k[0] = indexPubkeyPrefix
-		copy(k[1:], pubkey)
-		binary.BigEndian.PutUint32(k[1+32:], uint32(evt.CreatedAt))
-		copy(k[1+32+4:], idx)
-		keys = append(keys, k)
-	}
-
-	{
-		// ~ by kind+date
-		k := make([]byte, 1+2+4+4)
-		k[0] = indexKindPrefix
-		binary.BigEndian.PutUint16(k[1:], uint16(evt.Kind))
-		binary.BigEndian.PutUint32(k[1+2:], uint32(evt.CreatedAt))
-		copy(k[1+2+4:], idx)
-		keys = append(keys, k)
-	}
-
-	{
-		// ~ by pubkey+kind+date
-		pubkey, _ := hex.DecodeString(evt.PubKey)
-		k := make([]byte, 1+32+2+4+4)
-		k[0] = indexPubkeyKindPrefix
-		copy(k[1:], pubkey)
-		binary.BigEndian.PutUint16(k[1+32:], uint16(evt.Kind))
-		binary.BigEndian.PutUint32(k[1+32+2:], uint32(evt.CreatedAt))
-		copy(k[1+32+2+4:], idx)
-		keys = append(keys, k)
-	}
-
-	// ~ by tagvalue+date
-	for _, tag := range evt.Tags {
-		if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 {
-			continue
-		}
-
-		var v []byte
-		var indexPrefix byte
-		if vb, _ := hex.DecodeString(tag[1]); len(vb) == 32 {
-			// store value as bytes
-			v = vb
-			indexPrefix = indexTag32Prefix
-		} else {
-			v = []byte(tag[1])
-			indexPrefix = indexTagPrefix
-		}
-
-		k := make([]byte, 1+len(v)+4+4)
-		k[0] = indexPrefix
-		copy(k[1:], v)
-		binary.BigEndian.PutUint32(k[1+len(v):], uint32(evt.CreatedAt))
-		copy(k[1+len(v)+4:], idx)
-		keys = append(keys, k)
-	}
-
-	{
-		// ~ by date only
-		k := make([]byte, 1+4+4)
-		k[0] = indexCreatedAtPrefix
-		binary.BigEndian.PutUint32(k[1:], uint32(evt.CreatedAt))
-		copy(k[1+4:], idx)
-		keys = append(keys, k)
-	}
-
-	return keys
-}

badger/migrations.go

@@ -8,14 +8,21 @@ import (
 )
 
 func (b *BadgerBackend) runMigrations() error {
-	return b.View(func(txn *badger.Txn) error {
-		item, err := txn.Get([]byte{dbVersionKey})
-		if err != nil {
-			return err
-		}
-		item.Value(func(val []byte) error {
+	return b.Update(func(txn *badger.Txn) error {
 		var version uint16
 
+		item, err := txn.Get([]byte{dbVersionKey})
+		if err == badger.ErrKeyNotFound {
+			version = 0
+		} else if err != nil {
+			return err
+		} else {
+			item.Value(func(val []byte) error {
+				version = binary.BigEndian.Uint16(val)
+				return nil
+			})
+		}
+
 		// do the migrations in increasing steps (there is no rollback)
 		//
 
@@ -33,29 +40,68 @@ func (b *BadgerBackend) runMigrations() error {
 				item := it.Item()
 				key := item.Key()
 
-					if len(key) == 1+32+2+4 {
+				if len(key) == 1+32+4+4 {
 					// it's 32 bytes
 					log.Printf("moving key %x", key)
 					if err := txn.Delete(key); err != nil {
 						return err
 					}
 					key[0] = indexTag32Prefix
-						txn.Set(key, nil)
+					if err := txn.Set(key, nil); err != nil {
+						return err
+					}
 				}
 			}
 
 			// bump version
+			version = 1
 			if err := b.bumpVersion(txn, 1); err != nil {
 				return err
 			}
 		}
 
 		if version < 2 {
-				// ...
+			log.Println("migration 2: move all keys from indexTag to indexTagAddr if they are like 'a' tags")
+			prefix := []byte{indexTagPrefix}
+			it := txn.NewIterator(badger.IteratorOptions{
+				PrefetchValues: true,
+				PrefetchSize:   100,
+				Prefix:         prefix,
+			})
+			defer it.Close()
+
+			for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+				item := it.Item()
+				key := item.Key()
+
+				if kind, pkb, d := getAddrTagElements(string(key[1 : len(key)-4-4])); len(pkb) == 32 {
+					// it's an 'a' tag or alike
+					if err := txn.Delete(key); err != nil {
+						return err
+					}
+					k := make([]byte, 1+2+32+len(d)+4+4)
+					k[0] = indexTagAddrPrefix
+					binary.BigEndian.PutUint16(k[1:], kind)
+					copy(k[1+2:], pkb)
+					copy(k[1+2+32:], d)
+					copy(k[1+2+32+len(d):], key[len(key)-4-4:])
+					if err := txn.Set(k, nil); err != nil {
+						return err
+					}
+					log.Printf("moved key %x to %x", key, k)
+				}
 			}
 
-			return nil
-		})
+			// bump version
+			version = 2
+			if err := b.bumpVersion(txn, 2); err != nil {
+				return err
+			}
+		}
+
+		if version < 3 {
+			// ...
+		}
 
 		return nil
 	})

badger/query.go

@@ -265,21 +265,11 @@ func prepareQueries(filter nostr.Filter) (
 		i := 0
 		for _, values := range filter.Tags {
 			for _, value := range values {
-				bv, _ := hex.DecodeString(value)
-				var size int
-				if len(bv) == 32 {
-					// hex tag
-					size = 32
-					index = indexTag32Prefix
-				} else {
-					// string tag
-					bv = []byte(value)
-					size = len(bv)
-					index = indexTagPrefix
-				}
-				prefix := make([]byte, 1+size)
-				prefix[0] = index
-				copy(prefix[1:], bv)
+				// get key prefix (with full length) and offset where to write the last parts
+				k, offset := getTagIndexPrefix(value)
+				// remove the last parts part to get just the prefix we want here
+				prefix := k[0:offset]
+
 				queries[i] = query{i: i, prefix: prefix}
 				i++
 			}