diff --git a/badger/helpers.go b/badger/helpers.go new file mode 100644 index 0000000..0f50993 --- /dev/null +++ b/badger/helpers.go @@ -0,0 +1,125 @@ +package badger + +import ( + "encoding/binary" + "encoding/hex" + "strconv" + "strings" + + "github.com/nbd-wtf/go-nostr" +) + +func getAddrTagElements(tagValue string) (kind uint16, pkb []byte, d string) { + spl := strings.Split(tagValue, ":") + if len(spl) == 3 { + if pkb, _ := hex.DecodeString(spl[1]); len(pkb) == 32 { + if kind, err := strconv.ParseUint(spl[0], 10, 16); err == nil { + return uint16(kind), pkb, spl[2] + } + } + } + return 0, nil, "" +} + +func getTagIndexPrefix(tagValue string) ([]byte, int) { + var k []byte // the key with full length for created_at and idx at the end, but not filled with these + var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start + + if kind, pkb, d := getAddrTagElements(tagValue); len(pkb) == 32 { + // store value in the new special "a" tag index + k = make([]byte, 1+2+32+len(d)+4+4) + k[0] = indexTagAddrPrefix + binary.BigEndian.PutUint16(k[1:], kind) + copy(k[1+2:], pkb) + copy(k[1+2+32:], d) + offset = 1 + 2 + 32 + len(d) + } else if vb, _ := hex.DecodeString(tagValue); len(vb) == 32 { + // store value as bytes + k = make([]byte, 1+32+4+4) + k[0] = indexTag32Prefix + copy(k[1:], vb) + offset = 1 + 32 + } else { + // store whatever as utf-8 + k = make([]byte, 1+len(tagValue)+4+4) + k[0] = indexTagPrefix + copy(k[1:], tagValue) + offset = 1 + len(tagValue) + } + + return k, offset +} + +func getIndexKeysForEvent(evt *nostr.Event, idx []byte) [][]byte { + keys := make([][]byte, 0, 18) + + // indexes + { + // ~ by id + id, _ := hex.DecodeString(evt.ID) + k := make([]byte, 1+32+4) + k[0] = indexIdPrefix + copy(k[1:], id) + copy(k[1+32:], idx) + keys = append(keys, k) + } + + { + // ~ by pubkey+date + pubkey, _ := hex.DecodeString(evt.PubKey) + k := make([]byte, 1+32+4+4) + k[0] = indexPubkeyPrefix + copy(k[1:], pubkey) + binary.BigEndian.PutUint32(k[1+32:], uint32(evt.CreatedAt)) + copy(k[1+32+4:], idx) + keys = append(keys, k) + } + + { + // ~ by kind+date + k := make([]byte, 1+2+4+4) + k[0] = indexKindPrefix + binary.BigEndian.PutUint16(k[1:], uint16(evt.Kind)) + binary.BigEndian.PutUint32(k[1+2:], uint32(evt.CreatedAt)) + copy(k[1+2+4:], idx) + keys = append(keys, k) + } + + { + // ~ by pubkey+kind+date + pubkey, _ := hex.DecodeString(evt.PubKey) + k := make([]byte, 1+32+2+4+4) + k[0] = indexPubkeyKindPrefix + copy(k[1:], pubkey) + binary.BigEndian.PutUint16(k[1+32:], uint16(evt.Kind)) + binary.BigEndian.PutUint32(k[1+32+2:], uint32(evt.CreatedAt)) + copy(k[1+32+2+4:], idx) + keys = append(keys, k) + } + + // ~ by tagvalue+date + for _, tag := range evt.Tags { + if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 { + continue + } + + // get key prefix (with full length) and offset where to write the last parts + k, offset := getTagIndexPrefix(tag[1]) + + // write the last parts (created_at and idx) + binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt)) + copy(k[offset+4:], idx) + keys = append(keys, k) + } + + { + // ~ by date only + k := make([]byte, 1+4+4) + k[0] = indexCreatedAtPrefix + binary.BigEndian.PutUint32(k[1:], uint32(evt.CreatedAt)) + copy(k[1+4:], idx) + keys = append(keys, k) + } + + return keys +} diff --git a/badger/lib.go b/badger/lib.go index 4b4a6f4..4cdd6d1 100644 --- a/badger/lib.go +++ b/badger/lib.go @@ -2,11 +2,10 @@ package badger import ( "encoding/binary" - "encoding/hex" + "fmt" "github.com/dgraph-io/badger/v4" "github.com/fiatjaf/eventstore" - "github.com/nbd-wtf/go-nostr" ) const ( @@ -19,6 +18,7 @@ const ( indexPubkeyKindPrefix byte = 5 indexTagPrefix byte = 6 indexTag32Prefix byte = 7 + indexTagAddrPrefix byte = 8 ) var _ eventstore.Store = (*BadgerBackend)(nil) @@ -42,6 +42,10 @@ func (b *BadgerBackend) Init() error { return err } + if err := b.runMigrations(); err != nil { + return fmt.Errorf("error running migrations: %w", err) + } + if b.MaxLimit == 0 { b.MaxLimit = 500 } @@ -61,87 +65,3 @@ func (b BadgerBackend) Serial() []byte { binary.BigEndian.PutUint32(vb[1:], uint32(v)) return vb } - -func getIndexKeysForEvent(evt *nostr.Event, idx []byte) [][]byte { - keys := make([][]byte, 0, 18) - - // indexes - { - // ~ by id - id, _ := hex.DecodeString(evt.ID) - k := make([]byte, 1+32+4) - k[0] = indexIdPrefix - copy(k[1:], id) - copy(k[1+32:], idx) - keys = append(keys, k) - } - - { - // ~ by pubkey+date - pubkey, _ := hex.DecodeString(evt.PubKey) - k := make([]byte, 1+32+4+4) - k[0] = indexPubkeyPrefix - copy(k[1:], pubkey) - binary.BigEndian.PutUint32(k[1+32:], uint32(evt.CreatedAt)) - copy(k[1+32+4:], idx) - keys = append(keys, k) - } - - { - // ~ by kind+date - k := make([]byte, 1+2+4+4) - k[0] = indexKindPrefix - binary.BigEndian.PutUint16(k[1:], uint16(evt.Kind)) - binary.BigEndian.PutUint32(k[1+2:], uint32(evt.CreatedAt)) - copy(k[1+2+4:], idx) - keys = append(keys, k) - } - - { - // ~ by pubkey+kind+date - pubkey, _ := hex.DecodeString(evt.PubKey) - k := make([]byte, 1+32+2+4+4) - k[0] = indexPubkeyKindPrefix - copy(k[1:], pubkey) - binary.BigEndian.PutUint16(k[1+32:], uint16(evt.Kind)) - binary.BigEndian.PutUint32(k[1+32+2:], uint32(evt.CreatedAt)) - copy(k[1+32+2+4:], idx) - keys = append(keys, k) - } - - // ~ by tagvalue+date - for _, tag := range evt.Tags { - if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 { - continue - } - - var v []byte - var indexPrefix byte - if vb, _ := hex.DecodeString(tag[1]); len(vb) == 32 { - // store value as bytes - v = vb - indexPrefix = indexTag32Prefix - } else { - v = []byte(tag[1]) - indexPrefix = indexTagPrefix - } - - k := make([]byte, 1+len(v)+4+4) - k[0] = indexPrefix - copy(k[1:], v) - binary.BigEndian.PutUint32(k[1+len(v):], uint32(evt.CreatedAt)) - copy(k[1+len(v)+4:], idx) - keys = append(keys, k) - } - - { - // ~ by date only - k := make([]byte, 1+4+4) - k[0] = indexCreatedAtPrefix - binary.BigEndian.PutUint32(k[1:], uint32(evt.CreatedAt)) - copy(k[1+4:], idx) - keys = append(keys, k) - } - - return keys -} diff --git a/badger/migrations.go b/badger/migrations.go index 8236bf8..32dc631 100644 --- a/badger/migrations.go +++ b/badger/migrations.go @@ -8,54 +8,100 @@ import ( ) func (b *BadgerBackend) runMigrations() error { - return b.View(func(txn *badger.Txn) error { + return b.Update(func(txn *badger.Txn) error { + var version uint16 + item, err := txn.Get([]byte{dbVersionKey}) - if err != nil { + if err == badger.ErrKeyNotFound { + version = 0 + } else if err != nil { return err + } else { + item.Value(func(val []byte) error { + version = binary.BigEndian.Uint16(val) + return nil + }) } - item.Value(func(val []byte) error { - var version uint16 - // do the migrations in increasing steps (there is no rollback) - // + // do the migrations in increasing steps (there is no rollback) + // - if version < 1 { - log.Println("migration 1: move all keys from indexTag to indexTag32 if they are 32-bytes") - prefix := []byte{indexTagPrefix} - it := txn.NewIterator(badger.IteratorOptions{ - PrefetchValues: true, - PrefetchSize: 100, - Prefix: prefix, - }) - defer it.Close() + if version < 1 { + log.Println("migration 1: move all keys from indexTag to indexTag32 if they are 32-bytes") + prefix := []byte{indexTagPrefix} + it := txn.NewIterator(badger.IteratorOptions{ + PrefetchValues: true, + PrefetchSize: 100, + Prefix: prefix, + }) + defer it.Close() - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - item := it.Item() - key := item.Key() + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + item := it.Item() + key := item.Key() - if len(key) == 1+32+2+4 { - // it's 32 bytes - log.Printf("moving key %x", key) - if err := txn.Delete(key); err != nil { - return err - } - key[0] = indexTag32Prefix - txn.Set(key, nil) + if len(key) == 1+32+4+4 { + // it's 32 bytes + log.Printf("moving key %x", key) + if err := txn.Delete(key); err != nil { + return err + } + key[0] = indexTag32Prefix + if err := txn.Set(key, nil); err != nil { + return err } } + } - // bump version - if err := b.bumpVersion(txn, 1); err != nil { - return err + // bump version + version = 1 + if err := b.bumpVersion(txn, 1); err != nil { + return err + } + } + + if version < 2 { + log.Println("migration 2: move all keys from indexTag to indexTagAddr if they are like 'a' tags") + prefix := []byte{indexTagPrefix} + it := txn.NewIterator(badger.IteratorOptions{ + PrefetchValues: true, + PrefetchSize: 100, + Prefix: prefix, + }) + defer it.Close() + + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + item := it.Item() + key := item.Key() + + if kind, pkb, d := getAddrTagElements(string(key[1 : len(key)-4-4])); len(pkb) == 32 { + // it's an 'a' tag or alike + if err := txn.Delete(key); err != nil { + return err + } + k := make([]byte, 1+2+32+len(d)+4+4) + k[0] = indexTagAddrPrefix + binary.BigEndian.PutUint16(k[1:], kind) + copy(k[1+2:], pkb) + copy(k[1+2+32:], d) + copy(k[1+2+32+len(d):], key[len(key)-4-4:]) + if err := txn.Set(k, nil); err != nil { + return err + } + log.Printf("moved key %x to %x", key, k) } } - if version < 2 { - // ... + // bump version + version = 2 + if err := b.bumpVersion(txn, 2); err != nil { + return err } + } - return nil - }) + if version < 3 { + // ... + } return nil }) diff --git a/badger/query.go b/badger/query.go index 2418a3b..4302026 100644 --- a/badger/query.go +++ b/badger/query.go @@ -265,21 +265,11 @@ func prepareQueries(filter nostr.Filter) ( i := 0 for _, values := range filter.Tags { for _, value := range values { - bv, _ := hex.DecodeString(value) - var size int - if len(bv) == 32 { - // hex tag - size = 32 - index = indexTag32Prefix - } else { - // string tag - bv = []byte(value) - size = len(bv) - index = indexTagPrefix - } - prefix := make([]byte, 1+size) - prefix[0] = index - copy(prefix[1:], bv) + // get key prefix (with full length) and offset where to write the last parts + k, offset := getTagIndexPrefix(value) + // remove the last parts part to get just the prefix we want here + prefix := k[0:offset] + queries[i] = query{i: i, prefix: prefix} i++ }