diff --git a/database/fulltext.go b/database/fulltext.go index 216cc9d..3273e18 100644 --- a/database/fulltext.go +++ b/database/fulltext.go @@ -63,7 +63,7 @@ func (d *D) GetWordsFromContent(ev *event.E) (wordMap map[string]int) { !IsEntity(w) && !bytes.Contains(w, []byte(".")) { if len(w) == 64 || len(w) == 128 { - if _, err := hex.Dec(string(w)); !chk.E(err) { + if _, err := hex.Dec(string(w)); err == nil { continue } } diff --git a/database/get-event-indexes.go b/database/get-event-indexes.go index e28730a..700e8ef 100644 --- a/database/get-event-indexes.go +++ b/database/get-event-indexes.go @@ -164,6 +164,9 @@ func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *varint.V, err e continue } ph := pubhash.New() + if len(pkb) == 0 { + continue + } if err = ph.FromPubkey(pkb); chk.E(err) { err = nil continue diff --git a/database/get-event-indexes_test.go b/database/get-event-indexes_test.go index 23570fc..7fc911c 100644 --- a/database/get-event-indexes_test.go +++ b/database/get-event-indexes_test.go @@ -88,7 +88,7 @@ func TestGetEventIndexes(t *testing.T) { // check the event encodes to binary, decodes, and produces the identical canonical form binE := new(bytes.Buffer) if err = ev.MarshalWrite(binE); chk.E(err) { - // log.I.F("bogus tags probably: %s", b) + log.I.F("bogus tags probably: %s", b) encErrs++ // events that marshal with errors have e and p tag values that aren't hex and should not be accepted continue diff --git a/database/store-find_test.go b/database/store-find_test.go index b51850a..2fa296c 100644 --- a/database/store-find_test.go +++ b/database/store-find_test.go @@ -50,7 +50,7 @@ func TestD_StoreEvent(t *testing.T) { count++ if count%1000 == 0 { log.I.F("unmarshaled %d events", count) - break + // break } if err = d.StoreEvent(ev); chk.E(err) { continue diff --git a/event/binary.go b/event/binary.go index a885c41..c7dd342 100644 --- a/event/binary.go +++ b/event/binary.go @@ -5,7 +5,6 @@ import ( "x.realy.lol/chk" "x.realy.lol/ec/schnorr" - "x.realy.lol/errorf" "x.realy.lol/hex" "x.realy.lol/timestamp" "x.realy.lol/varint" @@ -49,14 +48,20 @@ func (ev *E) MarshalWrite(w io.Writer) (err error) { for i, y := range x { if i == 1 && isBin { var b []byte - b, err = hex.Dec(y) - if err != nil { - err = errorf.E("e or p tag value not hex: %s", err.Error()) - return - } - if len(b) != 32 { - err = errorf.E("e or p tag value with invalid decoded byte length %d", len(b)) - return + if b, err = hex.Dec(y); err != nil { + b = []byte(y) + // non-hex "p" or "e" tags have a 1 prefix to indicate not to hex decode. + _, _ = w.Write([]byte{1}) + err = nil + } else { + if len(b) != 32 { + // err = errorf.E("e or p tag value with invalid decoded byte length %d '%0x'", len(b), b) + b = []byte(y) + _, _ = w.Write([]byte{1}) + } else { + // hex values have a 2 prefix + _, _ = w.Write([]byte{2}) + } } varint.Encode(w, uint64(len(b))) _, _ = w.Write(b) @@ -111,6 +116,14 @@ func (ev *E) UnmarshalRead(r io.Reader) (err error) { var t []string var isBin bool for i := range nField { + var pr byte + if i == 1 && isBin { + prf := make([]byte, 1) + if _, err = r.Read(prf); chk.E(err) { + return + } + pr = prf[0] + } var lenField uint64 if lenField, err = varint.Decode(r); chk.E(err) { return @@ -119,16 +132,18 @@ func (ev *E) UnmarshalRead(r io.Reader) (err error) { if _, err = r.Read(field); chk.E(err) { return } - // if it is first field, length 1 and is e or p, the value field must be binary + // if it is first field, length 1 and is e or p, the value field should be binary if i == 0 && len(field) == 1 && (field[0] == 'e' || field[0] == 'p') { isBin = true } if i == 1 && isBin { - // this is a binary value, was an e or p tag key, 32 bytes long, encode value - // field to hex - f := make([]byte, 64) - _ = hex.EncBytes(f, field) - field = f + if pr == 2 { + // this is a binary value, was an e or p tag key, 32 bytes long, encode + // value field to hex + f := make([]byte, 64) + _ = hex.EncBytes(f, field) + field = f + } } t = append(t, string(field)) } diff --git a/tags/tags.go b/tags/tags.go index 9b458f2..92c4528 100644 --- a/tags/tags.go +++ b/tags/tags.go @@ -261,7 +261,7 @@ func (tags Tags) Get_a_Tags() (atags []Tag_a) { } // next must be a pubkey var pk []byte - if pk, err = hex.Dec(parts[1]); chk.E(err) { + if pk, err = hex.Dec(parts[1]); err != nil { continue } atag.Pubkey = pk