allow non-hex e and p tags

mostly because some clients use p tags in follow lists for whatever reason. follow lists are generally fetched by pubkey+kind so the missing index because it's not a pubkey is really nbd, and pubkey tag searches will also work with the kind.

database/fulltext.go

@@ -63,7 +63,7 @@ func (d *D) GetWordsFromContent(ev *event.E) (wordMap map[string]int) {
 				!IsEntity(w) &&
 				!bytes.Contains(w, []byte(".")) {
 				if len(w) == 64 || len(w) == 128 {
-					if _, err := hex.Dec(string(w)); !chk.E(err) {
+					if _, err := hex.Dec(string(w)); err == nil {
 						continue
 					}
 				}

database/get-event-indexes.go

@@ -164,6 +164,9 @@ func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *varint.V, err e
 			continue
 		}
 		ph := pubhash.New()
+		if len(pkb) == 0 {
+			continue
+		}
 		if err = ph.FromPubkey(pkb); chk.E(err) {
 			err = nil
 			continue

database/get-event-indexes_test.go

@@ -88,7 +88,7 @@ func TestGetEventIndexes(t *testing.T) {
 		// check the event encodes to binary, decodes, and produces the identical canonical form
 		binE := new(bytes.Buffer)
 		if err = ev.MarshalWrite(binE); chk.E(err) {
-			// log.I.F("bogus tags probably: %s", b)
+			log.I.F("bogus tags probably: %s", b)
 			encErrs++
 			// events that marshal with errors have e and p tag values that aren't hex and should not be accepted
 			continue

database/store-find_test.go

@@ -50,7 +50,7 @@ func TestD_StoreEvent(t *testing.T) {
 		count++
 		if count%1000 == 0 {
 			log.I.F("unmarshaled %d events", count)
-			break
+			// break
 		}
 		if err = d.StoreEvent(ev); chk.E(err) {
 			continue

event/binary.go

@@ -5,7 +5,6 @@ import (
 
 	"x.realy.lol/chk"
 	"x.realy.lol/ec/schnorr"
-	"x.realy.lol/errorf"
 	"x.realy.lol/hex"
 	"x.realy.lol/timestamp"
 	"x.realy.lol/varint"
@@ -49,14 +48,20 @@ func (ev *E) MarshalWrite(w io.Writer) (err error) {
 		for i, y := range x {
 			if i == 1 && isBin {
 				var b []byte
-				b, err = hex.Dec(y)
+				if b, err = hex.Dec(y); err != nil {
-				if err != nil {
+					b = []byte(y)
-					err = errorf.E("e or p tag value not hex: %s", err.Error())
+					// non-hex "p" or "e" tags have a 1 prefix to indicate not to hex decode.
-					return
+					_, _ = w.Write([]byte{1})
-				}
+					err = nil
-				if len(b) != 32 {
+				} else {
-					err = errorf.E("e or p tag value with invalid decoded byte length %d", len(b))
+					if len(b) != 32 {
-					return
+						// err = errorf.E("e or p tag value with invalid decoded byte length %d '%0x'", len(b), b)
+						b = []byte(y)
+						_, _ = w.Write([]byte{1})
+					} else {
+						// hex values have a 2 prefix
+						_, _ = w.Write([]byte{2})
+					}
 				}
 				varint.Encode(w, uint64(len(b)))
 				_, _ = w.Write(b)
@@ -111,6 +116,14 @@ func (ev *E) UnmarshalRead(r io.Reader) (err error) {
 		var t []string
 		var isBin bool
 		for i := range nField {
+			var pr byte
+			if i == 1 && isBin {
+				prf := make([]byte, 1)
+				if _, err = r.Read(prf); chk.E(err) {
+					return
+				}
+				pr = prf[0]
+			}
 			var lenField uint64
 			if lenField, err = varint.Decode(r); chk.E(err) {
 				return
@@ -119,16 +132,18 @@ func (ev *E) UnmarshalRead(r io.Reader) (err error) {
 			if _, err = r.Read(field); chk.E(err) {
 				return
 			}
-			// if it is first field, length 1 and is e or p, the value field must be binary
+			// if it is first field, length 1 and is e or p, the value field should be binary
 			if i == 0 && len(field) == 1 && (field[0] == 'e' || field[0] == 'p') {
 				isBin = true
 			}
 			if i == 1 && isBin {
-				// this is a binary value, was an e or p tag key, 32 bytes long, encode value
+				if pr == 2 {
-				// field to hex
+					// this is a binary value, was an e or p tag key, 32 bytes long, encode
-				f := make([]byte, 64)
+					// value field to hex
-				_ = hex.EncBytes(f, field)
+					f := make([]byte, 64)
-				field = f
+					_ = hex.EncBytes(f, field)
+					field = f
+				}
 			}
 			t = append(t, string(field))
 		}

tags/tags.go

@@ -261,7 +261,7 @@ func (tags Tags) Get_a_Tags() (atags []Tag_a) {
 				}
 				// next must be a pubkey
 				var pk []byte
-				if pk, err = hex.Dec(parts[1]); chk.E(err) {
+				if pk, err = hex.Dec(parts[1]); err != nil {
 					continue
 				}
 				atag.Pubkey = pk