mleku/x-realy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test index generation

Browse Source
This commit is contained in:
mleku
2025-06-08 07:25:42 +01:00
parent 8ac12ccf1d
commit 6ffc5b1de0
34 changed files with 1457 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
apputil/apputil.go Normal file
View File

@@ -0,0 +1,24 @@
package apputil
import (
"os"
"path/filepath"
)
// EnsureDir checks a file could be written to a path, creates the directories
// as needed
func EnsureDir(fileName string) {
dirName := filepath.Dir(fileName)
if _, serr := os.Stat(dirName); serr != nil {
merr := os.MkdirAll(dirName, os.ModePerm)
if merr != nil {
panic(merr)
}
}
}
// FileExists reports whether the named file or directory exists.
func FileExists(filePath string) bool {
_, e := os.Stat(filePath)
return e == nil
}

2
apputil/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package apputil provides some simple filesystem functions
package apputil

2
codec/codec.go
View File

@@ -5,6 +5,6 @@ import (
)
type I interface {
MarshalWrite(w io.Writer)
MarshalWrite(w io.Writer) (err error)
UnmarshalRead(r io.Reader) (err error)
}

115
database/fulltext.go Normal file
View File

@@ -0,0 +1,115 @@
package database
import (
"bytes"
"unicode"
"unicode/utf8"
"github.com/clipperhouse/uax29/words"
"x.realy.lol/chk"
"x.realy.lol/database/indexes"
"x.realy.lol/database/indexes/types/fulltext"
"x.realy.lol/database/indexes/types/serial"
"x.realy.lol/database/indexes/types/size"
"x.realy.lol/event"
"x.realy.lol/hex"
"x.realy.lol/kind"
)
type Words struct {
ser *serial.S
ev *event.E
wordMap map[string]int
}
func (d *D) GetFulltextKeys(ev *event.E, ser *serial.S) (keys [][]byte, err error) {
w := d.GetWordsFromContent(ev)
for i := range w {
ft := fulltext.New()
ft.FromWord([]byte(i))
pos := size.New()
pos.FromUint32(uint32(w[i]))
buf := new(bytes.Buffer)
if err = indexes.FullTextWordEnc(ft, pos, ser).MarshalWrite(buf); chk.E(err) {
return
}
keys = append(keys, buf.Bytes())
}
return
}
func (d *D) GetWordsFromContent(ev *event.E) (wordMap map[string]int) {
wordMap = make(map[string]int)
if kind.IsText(ev.Kind) {
content := ev.Content
seg := words.NewSegmenter([]byte(content))
var counter int
for seg.Next() {
w := seg.Bytes()
w = bytes.ToLower(w)
var ru rune
ru, _ = utf8.DecodeRune(w)
// ignore the most common things that aren't words
if !unicode.IsSpace(ru) &&
!unicode.IsPunct(ru) &&
!unicode.IsSymbol(ru) &&
!bytes.HasSuffix(w, []byte(".jpg")) &&
!bytes.HasSuffix(w, []byte(".png")) &&
!bytes.HasSuffix(w, []byte(".jpeg")) &&
!bytes.HasSuffix(w, []byte(".mp4")) &&
!bytes.HasSuffix(w, []byte(".mov")) &&
!bytes.HasSuffix(w, []byte(".aac")) &&
!bytes.HasSuffix(w, []byte(".mp3")) &&
!IsEntity(w) &&
!bytes.Contains(w, []byte(".")) {
if len(w) == 64 || len(w) == 128 {
if _, err := hex.Dec(string(w)); !chk.E(err) {
continue
}
}
wordMap[string(w)] = counter
counter++
}
}
content = content[:0]
}
return
}
func IsEntity(w []byte) (is bool) {
var b []byte
b = []byte("nostr:")
if bytes.Contains(w, b) && len(b)+10 < len(w) {
return true
}
b = []byte("npub")
if bytes.Contains(w, b) && len(b)+5 < len(w) {
return true
}
b = []byte("nsec")
if bytes.Contains(w, b) && len(b)+5 < len(w) {
return true
}
b = []byte("nevent")
if bytes.Contains(w, b) && len(b)+5 < len(w) {
return true
}
b = []byte("naddr")
if bytes.Contains(w, b) && len(b)+5 < len(w) {
return true
}
b = []byte("note")
if bytes.Contains(w, b) && len(b)+20 < len(w) {
return true
}
b = []byte("lnurl")
if bytes.Contains(w, b) && len(b)+20 < len(w) {
return true
}
b = []byte("cashu")
if bytes.Contains(w, b) && len(b)+20 < len(w) {
return true
}
return
}

263
database/get-event-indexes.go Normal file
View File

@@ -0,0 +1,263 @@
package database
import (
"bytes"
"time"
"x.realy.lol/chk"
"x.realy.lol/database/indexes"
"x.realy.lol/database/indexes/types/fullid"
identhash "x.realy.lol/database/indexes/types/identHash"
"x.realy.lol/database/indexes/types/idhash"
"x.realy.lol/database/indexes/types/kindidx"
"x.realy.lol/database/indexes/types/letter"
"x.realy.lol/database/indexes/types/pubhash"
"x.realy.lol/database/indexes/types/serial"
"x.realy.lol/database/indexes/types/timestamp"
"x.realy.lol/event"
"x.realy.lol/hex"
"x.realy.lol/tags"
)
// GetEventIndexes generates a set of indexes for a new event record. The first record is the
// key that should have the binary encoded event as its value.
func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, err error) {
// log.I.F("getting event indices for\n%s", ev.Serialize())
// get a new serial
ser := serial.New()
var s uint64
if s, err = d.Serial(); chk.E(err) {
return
}
ser.FromSerial(s)
// create the event id key
id := idhash.New()
var idb []byte
if idb, err = ev.IdBytes(); chk.E(err) {
return
}
if err = id.FromId(idb); chk.E(err) {
return
}
evIDB := new(bytes.Buffer)
if err = indexes.IdEnc(id, ser).MarshalWrite(evIDB); chk.E(err) {
return
}
indices = append(indices, evIDB.Bytes())
// create the full index key
fid := fullid.New()
if err = fid.FromId(idb); chk.E(err) {
return
}
p := pubhash.New()
var pk []byte
if pk, err = ev.PubBytes(); chk.E(err) {
return
}
if err = p.FromPubkey(pk); chk.E(err) {
return
}
ki := kindidx.FromKind(ev.Kind)
ca := &timestamp.T{}
ca.FromInt64(int64(ev.CreatedAt))
evIFiB := new(bytes.Buffer)
if err = indexes.FullIndexEnc(fid, p, ki, ca, ser).MarshalWrite(evIFiB); chk.E(err) {
return
}
indices = append(indices, evIFiB.Bytes())
// pubkey index
evIPkB := new(bytes.Buffer)
if err = indexes.PubkeyEnc(p, ser).MarshalWrite(evIPkB); chk.E(err) {
return
}
indices = append(indices, evIPkB.Bytes())
// pubkey-created_at index
evIPkCaB := new(bytes.Buffer)
if err = indexes.PubkeyCreatedAtEnc(p, ca, ser).MarshalWrite(evIPkCaB); chk.E(err) {
return
}
indices = append(indices, evIPkCaB.Bytes())
// created_at index
evICaB := new(bytes.Buffer)
if err = indexes.CreatedAtEnc(ca, ser).MarshalWrite(evICaB); chk.E(err) {
return
}
indices = append(indices, evICaB.Bytes())
// FirstSeen index
evIFsB := new(bytes.Buffer)
fs := &timestamp.T{}
fs.FromInt64(time.Now().Unix())
if err = indexes.FirstSeenEnc(ser, fs).MarshalWrite(evIFsB); chk.E(err) {
return
}
indices = append(indices, evIFsB.Bytes())
// Kind index
evIKiB := new(bytes.Buffer)
if err = indexes.KindEnc(ki, ser).MarshalWrite(evIKiB); chk.E(err) {
return
}
indices = append(indices, evIKiB.Bytes())
// tags
// TagA index
var atags []tags.Tag_a
var tagAs []indexes.TagA
atags = ev.Tags.Get_a_Tags()
for _, v := range atags {
aki, apk, aid, _ := indexes.TagAVars()
aki.Set(v.Kind)
if err = apk.FromPubkey(v.Pubkey); chk.E(err) {
continue
}
if err = aid.FromIdent([]byte(v.Ident)); chk.E(err) {
continue
}
tagAs = append(tagAs, indexes.TagA{
Ki: aki, P: apk, Id: aid, Ser: ser,
})
}
for _, v := range tagAs {
evITaB := new(bytes.Buffer)
if err = indexes.TagAEnc(v.Ki, v.P, v.Id, ser).MarshalWrite(evITaB); chk.E(err) {
return
}
indices = append(indices, evITaB.Bytes())
}
// TagEvent index
eTags := ev.Tags.GetAllExactKeys("e")
for _, v := range eTags {
eid := v.Value()
var eh []byte
if eh, err = hex.Dec(eid); chk.E(err) {
err = nil
continue
}
ih := idhash.New()
if err = ih.FromId(eh); chk.E(err) {
err = nil
continue
}
evIeB := new(bytes.Buffer)
if err = indexes.TagEventEnc(ih, ser).MarshalWrite(evIeB); chk.E(err) {
return
}
indices = append(indices, evIeB.Bytes())
}
// TagPubkey index
pTags := ev.Tags.GetAllExactKeys("p")
for _, v := range pTags {
pt := v.Value()
var pkb []byte
if pkb, err = hex.Dec(pt); err != nil {
err = nil
continue
}
ph := pubhash.New()
if err = ph.FromPubkey(pkb); chk.E(err) {
err = nil
continue
}
evIpB := new(bytes.Buffer)
if err = indexes.TagPubkeyEnc(ph, ser).MarshalWrite(evIpB); chk.E(err) {
return
}
indices = append(indices, evIpB.Bytes())
}
// TagHashtag index
ttags := ev.Tags.GetAllExactKeys("t")
for _, v := range ttags {
ht := v.Value()
hh := identhash.New()
if err = hh.FromIdent([]byte(ht)); chk.E(err) {
err = nil
continue
}
evIhB := new(bytes.Buffer)
if err = indexes.TagHashtagEnc(hh, ser).MarshalWrite(evIhB); chk.E(err) {
return
}
indices = append(indices, evIhB.Bytes())
}
// TagIdentifier index
dtags := ev.Tags.GetAllExactKeys("d")
for _, v := range dtags {
dt := v.Value()
dh := identhash.New()
if err = dh.FromIdent([]byte(dt)); chk.E(err) {
err = nil
continue
}
evIidB := new(bytes.Buffer)
if err = indexes.TagIdentifierEnc(dh, ser).MarshalWrite(evIidB); chk.E(err) {
return
}
indices = append(indices, evIidB.Bytes())
}
// TagLetter index, TagProtected, TagNonstandard
for _, v := range ev.Tags {
key := v.Key()
if len(key) == 1 {
switch key {
case "t", "p", "e":
// we already made indexes for these letters
continue
case "-":
// TagProtected
evIprotB := new(bytes.Buffer)
if err = indexes.TagProtectedEnc(p, ser).MarshalWrite(evIprotB); chk.E(err) {
return
}
indices = append(indices, evIprotB.Bytes())
default:
if !((key[0] >= 'a' && key[0] <= 'z') || (key[0] >= 'A' && key[0] <= 'Z')) {
// this is not a single letter tag or protected. nonstandard
nk, nv := identhash.New(), identhash.New()
_ = nk.FromIdent([]byte(key))
if len(v) > 1 {
_ = nv.FromIdent([]byte(v.Value()))
} else {
_ = nv.FromIdent([]byte{})
}
evInsB := new(bytes.Buffer)
if err = indexes.TagNonstandardEnc(nk, nv, ser).MarshalWrite(evInsB); chk.E(err) {
return
}
indices = append(indices, evInsB.Bytes())
continue
}
}
// we have a single letter that is not e, p or t
l := letter.New(key[0])
val := identhash.New()
// this can be empty, but the hash would still be distinct
if err = val.FromIdent([]byte(v.Value())); chk.E(err) {
continue
}
evIlB := new(bytes.Buffer)
if err = indexes.TagLetterEnc(l, val, ser).MarshalWrite(evIlB); chk.E(err) {
return
}
indices = append(indices, evIlB.Bytes())
} else {
// TagNonstandard
nk, nv := identhash.New(), identhash.New()
_ = nk.FromIdent([]byte(key))
if len(v) > 1 {
_ = nv.FromIdent([]byte(v.Value()))
} else {
_ = nv.FromIdent([]byte{})
}
evInsB := new(bytes.Buffer)
if err = indexes.TagNonstandardEnc(nk, nv, ser).MarshalWrite(evInsB); chk.E(err) {
return
}
indices = append(indices, evInsB.Bytes())
}
}
// FullTextWord index
var ftk [][]byte
if ftk, err = d.GetFulltextKeys(ev, ser); chk.E(err) {
return
}
indices = append(indices, ftk...)
return
}

142
database/get-event-indexes_test.go Normal file
View File

@@ -0,0 +1,142 @@
package database
import (
"bufio"
"bytes"
_ "embed"
"io"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"x.realy.lol/apputil"
"x.realy.lol/chk"
"x.realy.lol/event"
"x.realy.lol/log"
"x.realy.lol/units"
)
var ExampleEvents []byte
func init() {
var err error
if !apputil.FileExists("examples.jsonl") {
var req *http.Request
req, err = http.NewRequest("GET", "https://files.mleku.dev/examples.jsonl", nil)
if err != nil {
panic("wtf")
}
var res *http.Response
if res, err = http.DefaultClient.Do(req); chk.E(err) {
panic("wtf")
}
var fh *os.File
if fh, err = os.OpenFile("examples.jsonl", os.O_CREATE|os.O_RDWR, 0600); chk.E(err) {
panic("wtf")
}
if _, err = io.Copy(fh, res.Body); chk.E(err) {
panic("wtf")
}
res.Body.Close()
}
log.I.F("loading file...")
var oh *os.File
if oh, err = os.Open("examples.jsonl"); chk.E(err) {
panic("wtf")
}
buf := new(bytes.Buffer)
if _, err = io.Copy(buf, oh); chk.E(err) {
panic("wtf")
}
ExampleEvents = buf.Bytes()
oh.Close()
}
func TestGetEventIndexes(t *testing.T) {
var err error
d := New()
tmpDir := filepath.Join(os.TempDir(), "testrealy")
if err = d.Init(tmpDir); chk.E(err) {
t.Fatal(err)
}
defer d.Close()
defer os.RemoveAll(tmpDir)
buf := bytes.NewBuffer(ExampleEvents)
scan := bufio.NewScanner(buf)
scan.Buffer(make([]byte, 5120000), 5120000)
var count, errs, encErrs, datasize, size, binsize int
start := time.Now()
for scan.Scan() {
b := scan.Bytes()
ev := event.New()
if err = ev.Unmarshal(b); chk.E(err) {
t.Fatalf("%s:\n%s", err, b)
}
// verify the signature on the event
var ok bool
if ok, err = ev.Verify(); chk.E(err) {
errs++
continue
}
if !ok {
errs++
log.E.F("event signature is invalid\n%s", b)
continue
}
// check the event encodes to binary, decodes, and produces the identical canonical form
binE := new(bytes.Buffer)
if err = ev.MarshalWrite(binE); chk.E(err) {
// log.I.F("bogus tags probably: %s", b)
encErrs++
// events that marshal with errors have e and p tag values that aren't hex and should not be accepted
continue
}
ev2 := event.New()
bin2 := bytes.NewBuffer(binE.Bytes())
if err = ev2.UnmarshalRead(bin2); chk.E(err) {
encErrs++
continue
}
var can1, can2 []byte
ev.ToCanonical(can1)
ev2.ToCanonical(can2)
if !bytes.Equal(can1, can2) {
encErrs++
log.I.S(can1, can2)
continue
}
binsize += len(binE.Bytes())
var valid bool
if valid, err = ev.Verify(); chk.E(err) {
log.I.F("%s", b)
encErrs++
continue
}
if !valid {
t.Fatalf("event failed to verify\n%s", b)
}
var indices [][]byte
if indices, err = d.GetEventIndexes(ev); chk.E(err) {
t.Fatal(err)
}
// log.I.F("%s", b)
// log.I.S(indices)
datasize += len(b)
for _, v := range indices {
size += len(v)
}
_ = indices
count++
// if count == 10000 {
// break
// }
}
log.I.F("unmarshaled, verified and indexed %d events in %s, %d Mb of indexes from %d Mb of events, %d Mb as binary, failed verify %d, failed encode %d", count, time.Now().Sub(start), size/units.Mb, datasize/units.Mb, binsize/units.Mb, errs, encErrs)
d.Close()
os.RemoveAll(tmpDir)
}
var _ = `wdawdad\nhttps://cdn.discordapp.com/attachments/1277777226397388800/1278018649860472874/grain.png?ex=66cf471e&is=66cdf59e&hm=790aced618bb517ebd560e1fd3def537351ef130e239c0ee86d43ff63c44a146&","sig":"2abc1b3bb119071209daba6bf2b6c76cdad036249aad624938a5a2736739d6c139adb7aa94d24550bc53a972e75c40549513a74d9ace8c4435a5d262c172300b`
var _ = ``

157
indexes/indexes.go → database/indexes/indexes.go
View File

@@ -5,18 +5,18 @@ import (
"x.realy.lol/chk"
"x.realy.lol/codec"
"x.realy.lol/indexes/prefixes"
"x.realy.lol/indexes/types/fullid"
"x.realy.lol/indexes/types/fulltext"
identhash "x.realy.lol/indexes/types/identHash"
"x.realy.lol/indexes/types/idhash"
"x.realy.lol/indexes/types/kindidx"
"x.realy.lol/indexes/types/letter"
"x.realy.lol/indexes/types/prefix"
"x.realy.lol/indexes/types/pubhash"
"x.realy.lol/indexes/types/serial"
"x.realy.lol/indexes/types/size"
"x.realy.lol/indexes/types/timestamp"
"x.realy.lol/database/indexes/prefixes"
"x.realy.lol/database/indexes/types/fullid"
"x.realy.lol/database/indexes/types/fulltext"
"x.realy.lol/database/indexes/types/identHash"
"x.realy.lol/database/indexes/types/idhash"
"x.realy.lol/database/indexes/types/kindidx"
"x.realy.lol/database/indexes/types/letter"
"x.realy.lol/database/indexes/types/prefix"
"x.realy.lol/database/indexes/types/pubhash"
"x.realy.lol/database/indexes/types/serial"
"x.realy.lol/database/indexes/types/size"
"x.realy.lol/database/indexes/types/timestamp"
)
type Encs []codec.I
@@ -31,10 +31,13 @@ type T struct {
// decode variant does not add the prefix encoder because it has been read by prefixes.Identify.
func New(encoders ...codec.I) (i *T) { return &T{encoders} }
func (t *T) MarshalWrite(w io.Writer) {
func (t *T) MarshalWrite(w io.Writer) (err error) {
for _, e := range t.Encs {
e.MarshalWrite(w)
if err = e.MarshalWrite(w); chk.E(err) {
return
}
}
return
}
func (t *T) UnmarshalRead(r io.Reader) (err error) {
@@ -42,36 +45,35 @@ func (t *T) UnmarshalRead(r io.Reader) (err error) {
if err = e.UnmarshalRead(r); chk.E(err) {
return
}
// log.I.S(e)
}
return
}
func EventVars() (ser *serial.T) {
func EventVars() (ser *serial.S) {
ser = serial.New()
return
}
func EventEnc(ser *serial.T) (enc *T) {
func EventEnc(ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.Event), ser)
}
func EventDec(ser *serial.T) (enc *T) {
func EventDec(ser *serial.S) (enc *T) {
return New(prefix.New(), ser)
}
func IdVars() (id *idhash.T, ser *serial.T) {
func IdVars() (id *idhash.T, ser *serial.S) {
id = idhash.New()
ser = serial.New()
return
}
func IdEnc(id *idhash.T, ser *serial.T) (enc *T) {
func IdEnc(id *idhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.Id), id, ser)
}
func IdDec(id *idhash.T, ser *serial.T) (enc *T) {
func IdDec(id *idhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), id, ser)
}
func FullIndexVars() (t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T, ser *serial.T) {
ca *timestamp.T, ser *serial.S) {
t = fullid.New()
p = pubhash.New()
ki = kindidx.FromKind(0)
@@ -80,206 +82,213 @@ func FullIndexVars() (t *fullid.T, p *pubhash.T, ki *kindidx.T,
return
}
func FullIndexEnc(t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T, ser *serial.T) (enc *T) {
ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.FullIndex), t, p, ki, ca, ser)
}
func FullIndexDec(t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T, ser *serial.T) (enc *T) {
ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(), t, p, ki, ca, ser)
}
func PubkeyVars() (p *pubhash.T, ser *serial.T) {
func PubkeyVars() (p *pubhash.T, ser *serial.S) {
p = pubhash.New()
ser = serial.New()
return
}
func PubkeyEnc(p *pubhash.T, ser *serial.T) (enc *T) {
func PubkeyEnc(p *pubhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.Pubkey), p, ser)
}
func PubkeyDec(p *pubhash.T, ser *serial.T) (enc *T) {
func PubkeyDec(p *pubhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), p, ser)
}
func PubkeyCreatedAtVars() (p *pubhash.T, ca *timestamp.T, ser *serial.T) {
func PubkeyCreatedAtVars() (p *pubhash.T, ca *timestamp.T, ser *serial.S) {
p = pubhash.New()
ca = &timestamp.T{}
ser = serial.New()
return
}
func PubkeyCreatedAtEnc(p *pubhash.T, ca *timestamp.T, ser *serial.T) (enc *T) {
func PubkeyCreatedAtEnc(p *pubhash.T, ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.PubkeyCreatedAt), p, ca, ser)
}
func PubkeyCreatedAtDec(p *pubhash.T, ca *timestamp.T, ser *serial.T) (enc *T) {
func PubkeyCreatedAtDec(p *pubhash.T, ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(), p, ca, ser)
}
func CreatedAtVars() (ca *timestamp.T, ser *serial.T) {
func CreatedAtVars() (ca *timestamp.T, ser *serial.S) {
ca = &timestamp.T{}
ser = serial.New()
return
}
func CreatedAtEnc(ca *timestamp.T, ser *serial.T) (enc *T) {
func CreatedAtEnc(ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.CreatedAt), ca, ser)
}
func CreatedAtDec(ca *timestamp.T, ser *serial.T) (enc *T) {
func CreatedAtDec(ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(), ca, ser)
}
func FirstSeenVars() (ser *serial.T, ts *timestamp.T) {
func FirstSeenVars() (ser *serial.S, ts *timestamp.T) {
ts = &timestamp.T{}
ser = serial.New()
return
}
func FirstSeenEnc(ser *serial.T, ts *timestamp.T) (enc *T) {
func FirstSeenEnc(ser *serial.S, ts *timestamp.T) (enc *T) {
return New(prefix.New(prefixes.FirstSeen), ser, ts)
}
func FirstSeenDec(ser *serial.T, ts *timestamp.T) (enc *T) {
func FirstSeenDec(ser *serial.S, ts *timestamp.T) (enc *T) {
return New(prefix.New(), ser, ts)
}
func KindVars() (ki *kindidx.T, ser *serial.T) {
func KindVars() (ki *kindidx.T, ser *serial.S) {
ki = kindidx.FromKind(0)
ser = serial.New()
return
}
func KindEnc(ki *kindidx.T, ser *serial.T) (enc *T) {
func KindEnc(ki *kindidx.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.Kind), ki, ser)
}
func KindDec(ki *kindidx.T, ser *serial.T) (enc *T) {
func KindDec(ki *kindidx.T, ser *serial.S) (enc *T) {
return New(prefix.New(), ki, ser)
}
func TagAVars() (ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.T) {
type TagA struct {
Ki *kindidx.T
P *pubhash.T
Id *identhash.T
Ser *serial.S
}
func TagAVars() (ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) {
ki = kindidx.FromKind(0)
p = pubhash.New()
id = identhash.New()
ser = serial.New()
return
}
func TagAEnc(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.T) (enc *T) {
func TagAEnc(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagA), ki, p, id, ser)
}
func TagADec(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.T) (enc *T) {
func TagADec(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), ki, p, id, ser)
}
func TagEventVars() (id *idhash.T, ser *serial.T) {
func TagEventVars() (id *idhash.T, ser *serial.S) {
id = idhash.New()
ser = serial.New()
return
}
func TagEventEnc(id *idhash.T, ser *serial.T) (enc *T) {
func TagEventEnc(id *idhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagEvent), id, ser)
}
func TagEventDec(id *idhash.T, ser *serial.T) (enc *T) {
func TagEventDec(id *idhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), id, ser)
}
func TagPubkeyVars() (p *pubhash.T, ser *serial.T) {
func TagPubkeyVars() (p *pubhash.T, ser *serial.S) {
p = pubhash.New()
ser = serial.New()
return
}
func TagPubkeyEnc(p *pubhash.T, ser *serial.T) (enc *T) {
func TagPubkeyEnc(p *pubhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagPubkey), p, ser)
}
func TagPubkeyDec(p *pubhash.T, ser *serial.T) (enc *T) {
func TagPubkeyDec(p *pubhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), p, ser)
}
func TagHashtagVars() (hashtag *identhash.T, ser *serial.T) {
func TagHashtagVars() (hashtag *identhash.T, ser *serial.S) {
hashtag = identhash.New()
ser = serial.New()
return
}
func TagHashtagEnc(hashtag *identhash.T, ser *serial.T) (enc *T) {
return New(prefix.New(prefixes.TagA), hashtag, ser)
func TagHashtagEnc(hashtag *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagHashtag), hashtag, ser)
}
func TagHashtagDec(hashtag *identhash.T, ser *serial.T) (enc *T) {
func TagHashtagDec(hashtag *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), hashtag, ser)
}
func TagIdentifierVars() (ident *identhash.T, ser *serial.T) {
func TagIdentifierVars() (ident *identhash.T, ser *serial.S) {
ident = identhash.New()
ser = serial.New()
return
}
func TagIdentifierEnc(ident *identhash.T, ser *serial.T) (enc *T) {
return New(prefix.New(prefixes.TagA), ident, ser)
func TagIdentifierEnc(ident *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagIdentifier), ident, ser)
}
func TagIdentifierDec(ident *identhash.T, ser *serial.T) (enc *T) {
func TagIdentifierDec(ident *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), ident, ser)
}
func TagLetterVars() (l *letter.T, val *identhash.T, ser *serial.T) {
func TagLetterVars() (l *letter.T, val *identhash.T, ser *serial.S) {
l = letter.New(0)
val = identhash.New()
ser = serial.New()
return
}
func TagLetterEnc(l *letter.T, val *identhash.T, ser *serial.T) (enc *T) {
func TagLetterEnc(l *letter.T, val *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagLetter), l, val, ser)
}
func TagLetterDec(l *letter.T, val *identhash.T, ser *serial.T) (enc *T) {
func TagLetterDec(l *letter.T, val *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), l, val, ser)
}
func TagProtectedVars() (p *pubhash.T, ser *serial.T) {
func TagProtectedVars() (p *pubhash.T, ser *serial.S) {
p = pubhash.New()
ser = serial.New()
return
}
func TagProtectedEnc(p *pubhash.T, ser *serial.T) (enc *T) {
func TagProtectedEnc(p *pubhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagProtected), p, ser)
}
func TagProtectedDec(p *pubhash.T, ser *serial.T) (enc *T) {
func TagProtectedDec(p *pubhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), p, ser)
}
func TagNonstandardVars() (key, value *identhash.T, ser *serial.T) {
func TagNonstandardVars() (key, value *identhash.T, ser *serial.S) {
key = identhash.New()
value = identhash.New()
ser = serial.New()
return
}
func TagNonstandardEnc(key, value *identhash.T, ser *serial.T) (enc *T) {
func TagNonstandardEnc(key, value *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.TagNonstandard), key, value, ser)
}
func TagNonstandardDec(key, value *identhash.T, ser *serial.T) (enc *T) {
func TagNonstandardDec(key, value *identhash.T, ser *serial.S) (enc *T) {
return New(prefix.New(), key, value, ser)
}
func FullTextWordVars() (fw *fulltext.T, pos *size.T, ser *serial.T) {
func FullTextWordVars() (fw *fulltext.T, pos *size.T, ser *serial.S) {
fw = fulltext.New()
pos = size.New()
ser = serial.New()
return
}
func FullTextWordEnc(fw *fulltext.T, pos *size.T, ser *serial.T) (enc *T) {
func FullTextWordEnc(fw *fulltext.T, pos *size.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.FulltextWord), fw, pos, ser)
}
func FullTextWordDec(fw *fulltext.T, pos *size.T, ser *serial.T) (enc *T) {
func FullTextWordDec(fw *fulltext.T, pos *size.T, ser *serial.S) (enc *T) {
return New(prefix.New(), fw, pos, ser)
}
func LastAccessedVars() (ser *serial.T) {
func LastAccessedVars() (ser *serial.S) {
ser = serial.New()
return
}
func LastAccessedEnc(ser *serial.T) (enc *T) {
func LastAccessedEnc(ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.LastAccessed), ser)
}
func LastAccessedDec(ser *serial.T) (enc *T) {
func LastAccessedDec(ser *serial.S) (enc *T) {
return New(prefix.New(), ser)
}
func AccessCounterVars() (ser *serial.T) {
func AccessCounterVars() (ser *serial.S) {
ser = serial.New()
return
}
func AccessCounterEnc(ser *serial.T) (enc *T) {
func AccessCounterEnc(ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.AccessCounter), ser)
}
func AccessCounterDec(ser *serial.T) (enc *T) {
func AccessCounterDec(ser *serial.S) (enc *T) {
return New(prefix.New(), ser)
}

4
indexes/indexes_test.go → database/indexes/indexes_test.go
View File

@@ -10,9 +10,9 @@ import (
"lukechampine.com/frand"
"x.realy.lol/chk"
"x.realy.lol/database/indexes/prefixes"
"x.realy.lol/database/indexes/types/prefix"
"x.realy.lol/ec/schnorr"
"x.realy.lol/indexes/prefixes"
"x.realy.lol/indexes/types/prefix"
)
func TestEvent(t *testing.T) {

0
indexes/prefixes/prefixes.go → database/indexes/prefixes/prefixes.go
View File

0
database/readme.md → database/indexes/readme.md
View File

5
indexes/types/fullid/fullid.go → database/indexes/types/fullid/fullid.go
View File

@@ -26,7 +26,10 @@ func (fi *T) FromId(id []byte) (err error) {
}
func (fi *T) Bytes() (b []byte) { return fi.val }
func (fi *T) MarshalWrite(w io.Writer) { _, _ = w.Write(fi.val) }
func (fi *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(fi.val)
return
}
func (fi *T) UnmarshalRead(r io.Reader) (err error) {
if len(fi.val) < Len {

5
indexes/types/fulltext/fulltext.go → database/indexes/types/fulltext/fulltext.go
View File

@@ -17,9 +17,10 @@ func (ft *T) FromWord(word []byte) { ft.val = word }
func (ft *T) Bytes() (b []byte) { return ft.val }
func (ft *T) MarshalWrite(w io.Writer) {
func (ft *T) MarshalWrite(w io.Writer) (err error) {
varint.Encode(w, uint64(len(ft.val)))
_, _ = w.Write(ft.val)
_, err = w.Write(ft.val)
return
}
func (ft *T) UnmarshalRead(r io.Reader) (err error) {

5
indexes/types/identHash/identhash.go → database/indexes/types/identHash/identhash.go
View File

@@ -19,7 +19,10 @@ func (i *T) FromIdent(id []byte) (err error) {
func (i *T) Bytes() (b []byte) { return i.val }
func (i *T) MarshalWrite(w io.Writer) { _, _ = w.Write(i.val) }
func (i *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(i.val)
return
}
func (i *T) UnmarshalRead(r io.Reader) (err error) {
if len(i.val) < Len {

21
indexes/types/idhash/idhash.go → database/indexes/types/idhash/idhash.go
View File

@@ -5,8 +5,10 @@ import (
"github.com/minio/sha256-simd"
"x.realy.lol/chk"
"x.realy.lol/errorf"
"x.realy.lol/helpers"
"x.realy.lol/hex"
)
const Len = 8
@@ -24,9 +26,26 @@ func (i *T) FromId(id []byte) (err error) {
return
}
func (i *T) FromIdHex(idh string) (err error) {
var id []byte
if id, err = hex.Dec(idh); chk.E(err) {
return
}
if len(id) != sha256.Size {
err = errorf.E("invalid Id length, got %d require %d", len(id), sha256.Size)
return
}
i.val = helpers.Hash(id)[:Len]
return
}
func (i *T) Bytes() (b []byte) { return i.val }
func (i *T) MarshalWrite(w io.Writer) { _, _ = w.Write(i.val) }
func (i *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(i.val)
return
}
func (i *T) UnmarshalRead(r io.Reader) (err error) {
if len(i.val) < Len {

5
indexes/types/kindidx/kindidx.go → database/indexes/types/kindidx/kindidx.go
View File

@@ -37,7 +37,10 @@ func (k *T) ToKind() (kind int) {
}
func (k *T) Bytes() (b []byte) { return k.val }
func (k *T) MarshalWrite(w io.Writer) { _, _ = w.Write(k.val) }
func (k *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(k.val)
return
}
func (k *T) UnmarshalRead(r io.Reader) (err error) {
if len(k.val) < Len {

5
indexes/types/letter/letter.go → database/indexes/types/letter/letter.go
View File

@@ -16,7 +16,10 @@ func (p *T) Set(lb byte) { p.val = []byte{lb} }
func (p *T) Letter() byte { return p.val[0] }
func (p *T) MarshalWrite(w io.Writer) { _, _ = w.Write(p.val) }
func (p *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(p.val)
return
}
func (p *T) UnmarshalRead(r io.Reader) (err error) {
val := make([]byte, 1)

7
indexes/types/prefix/prefix.go → database/indexes/types/prefix/prefix.go
View File

@@ -3,7 +3,7 @@ package prefix
import (
"io"
"x.realy.lol/indexes/prefixes"
"x.realy.lol/database/indexes/prefixes"
)
const Len = 2
@@ -22,7 +22,10 @@ func New(prf ...int) (p *T) {
func (p *T) Bytes() (b []byte) { return p.val }
func (p *T) MarshalWrite(w io.Writer) { _, _ = w.Write(p.val) }
func (p *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(p.val)
return
}
func (p *T) UnmarshalRead(r io.Reader) (err error) {
_, err = r.Read(p.val)

5
indexes/types/pubhash/pubhash.go → database/indexes/types/pubhash/pubhash.go
View File

@@ -25,7 +25,10 @@ func (ph *T) FromPubkey(pk []byte) (err error) {
func (ph *T) Bytes() (b []byte) { return ph.val }
func (ph *T) MarshalWrite(w io.Writer) { _, _ = w.Write(ph.val) }
func (ph *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(ph.val)
return
}
func (ph *T) UnmarshalRead(r io.Reader) (err error) {
if len(ph.val) < Len {

21
indexes/types/serial/serial.go → database/indexes/types/serial/serial.go
View File

@@ -9,34 +9,37 @@ import (
const Len = 8
type T struct{ val []byte }
type S struct{ val []byte }
func New() (s *T) { return &T{make([]byte, Len)} }
func New() (s *S) { return &S{make([]byte, Len)} }
func (s *T) FromSerial(ser uint64) {
func (s *S) FromSerial(ser uint64) {
binary.LittleEndian.PutUint64(s.val, ser)
return
}
func FromBytes(ser []byte) (s *T, err error) {
func FromBytes(ser []byte) (s *S, err error) {
if len(ser) != Len {
err = errorf.E("serial must be %d bytes long, got %d", Len, len(ser))
return
}
s = &T{val: ser}
s = &S{val: ser}
return
}
func (s *T) ToSerial() (ser uint64) {
func (s *S) ToSerial() (ser uint64) {
ser = binary.LittleEndian.Uint64(s.val)
return
}
func (s *T) Bytes() (b []byte) { return s.val }
func (s *S) Bytes() (b []byte) { return s.val }
func (s *T) MarshalWrite(w io.Writer) { _, _ = w.Write(s.val) }
func (s *S) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(s.val)
return
}
func (s *T) UnmarshalRead(r io.Reader) (err error) {
func (s *S) UnmarshalRead(r io.Reader) (err error) {
if len(s.val) < Len {
s.val = make([]byte, Len)
} else {

5
indexes/types/size/serial.go → database/indexes/types/size/serial.go
View File

@@ -35,7 +35,10 @@ func (s *T) ToUint32() (ser uint32) {
func (s *T) Bytes() (b []byte) { return s.val }
func (s *T) MarshalWrite(w io.Writer) { _, _ = w.Write(s.val) }
func (s *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(s.val)
return
}
func (s *T) UnmarshalRead(r io.Reader) (err error) {
if len(s.val) < Len {

5
indexes/types/timestamp/timestamp.go → database/indexes/types/timestamp/timestamp.go
View File

@@ -32,7 +32,10 @@ func (ts *T) ToTimestamp() (timestamp timeStamp.Timestamp) {
}
func (ts *T) Bytes() (b []byte) { return ts.val }
func (ts *T) MarshalWrite(w io.Writer) { _, _ = w.Write(ts.val) }
func (ts *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(ts.val)
return
}
func (ts *T) UnmarshalRead(r io.Reader) (err error) {
if len(ts.val) < Len {

69
database/log.go Normal file
View File

@@ -0,0 +1,69 @@
package database
import (
"fmt"
"runtime"
"strings"
"x.realy.lol/atomic"
"x.realy.lol/log"
"x.realy.lol/lol"
)
// NewLogger creates a new badger logger.
func NewLogger(logLevel int, label string) (l *logger) {
log.T.Ln("getting logger for", label)
l = &logger{Label: label}
l.Level.Store(int32(logLevel))
return
}
type logger struct {
Level atomic.Int32
Label string
}
// SetLogLevel atomically adjusts the log level to the given log level code.
func (l *logger) SetLogLevel(level int) {
l.Level.Store(int32(level))
}
// Errorf is a log printer for this level of message.
func (l *logger) Errorf(s string, i ...interface{}) {
if l.Level.Load() >= lol.Error {
s = l.Label + ": " + s
txt := fmt.Sprintf(s, i...)
_, file, line, _ := runtime.Caller(2)
log.E.F("%s\n%s:%d", strings.TrimSpace(txt), file, line)
}
}
// Warningf is a log printer for this level of message.
func (l *logger) Warningf(s string, i ...interface{}) {
if l.Level.Load() >= lol.Warn {
s = l.Label + ": " + s
txt := fmt.Sprintf(s, i...)
_, file, line, _ := runtime.Caller(2)
log.D.F("%s\n%s:%d", strings.TrimSpace(txt), file, line)
}
}
// Infof is a log printer for this level of message.
func (l *logger) Infof(s string, i ...interface{}) {
if l.Level.Load() >= lol.Info {
s = l.Label + ": " + s
txt := fmt.Sprintf(s, i...)
_, file, line, _ := runtime.Caller(2)
log.D.F("%s\n%s:%d", strings.TrimSpace(txt), file, line)
}
}
// Debugf is a log printer for this level of message.
func (l *logger) Debugf(s string, i ...interface{}) {
if l.Level.Load() >= lol.Debug {
s = l.Label + ": " + s
txt := fmt.Sprintf(s, i...)
_, file, line, _ := runtime.Caller(2)
log.T.F("%s\n%s:%d", strings.TrimSpace(txt), file, line)
}
}

56
database/main.go
View File

@@ -1,4 +1,60 @@
package database
import (
"github.com/dgraph-io/badger/v4"
"x.realy.lol/chk"
"x.realy.lol/log"
"x.realy.lol/units"
)
type D struct {
dataDir string
BlockCacheSize int
Logger *logger
InitLogLevel int
// DB is the badger db
*badger.DB
// seq is the monotonic collision free index for raw event storage.
seq *badger.Sequence
}
func New() (d *D) {
d = &D{BlockCacheSize: units.Gb}
return
}
// Path returns the path where the database files are stored.
func (d *D) Path() string { return d.dataDir }
// Init sets up the database with the loaded configuration.
func (d *D) Init(path string) (err error) {
d.dataDir = path
log.I.Ln("opening realy event store at", d.dataDir)
opts := badger.DefaultOptions(d.dataDir)
opts.BlockCacheSize = int64(d.BlockCacheSize)
opts.BlockSize = units.Gb
opts.CompactL0OnClose = true
opts.LmaxCompaction = true
d.Logger = NewLogger(d.InitLogLevel, d.dataDir)
opts.Logger = d.Logger
if d.DB, err = badger.Open(opts); chk.E(err) {
return err
}
log.T.Ln("getting event store sequence index", d.dataDir)
if d.seq, err = d.DB.GetSequence([]byte("events"), 1000); chk.E(err) {
return err
}
return nil
}
func (d *D) Close() (err error) { return d.DB.Close() }
// Serial returns the next monotonic conflict free unique serial on the database.
func (d *D) Serial() (ser uint64, err error) {
if ser, err = d.seq.Next(); chk.E(err) {
}
// log.T.ToSliceOfBytes("serial %x", ser)
return
}

60
event/binary.go
View File

@@ -5,12 +5,17 @@ import (
"x.realy.lol/chk"
"x.realy.lol/ec/schnorr"
"x.realy.lol/errorf"
"x.realy.lol/hex"
"x.realy.lol/timestamp"
"x.realy.lol/varint"
)
// MarshalBinary writes a binary encoding of an event.
// todo: maybe we should make e and p tag values binary to reduce space usage
// MarshalWrite writes a binary encoding of an event.
//
// NOTE: Event must not be nil or this will panic. Use event.New.
//
// [ 32 bytes Id ]
// [ 32 bytes Pubkey ]
@@ -25,7 +30,10 @@ import (
//
// [ varint Content length ]
// [ 64 bytes Sig ]
func (ev *E) MarshalWrite(w io.Writer) {
func (ev *E) MarshalWrite(w io.Writer) (err error) {
if ev == nil {
panic("cannot marshal a nil event")
}
_, _ = w.Write(ev.GetIdBytes())
_, _ = w.Write(ev.GetPubkeyBytes())
varint.Encode(w, uint64(ev.CreatedAt))
@@ -33,18 +41,44 @@ func (ev *E) MarshalWrite(w io.Writer) {
varint.Encode(w, uint64(len(ev.Tags)))
for _, x := range ev.Tags {
varint.Encode(w, uint64(len(x)))
for _, y := range x {
varint.Encode(w, uint64(len(y)))
_, _ = w.Write([]byte(y))
// e and p tag values should be hex
var isBin bool
if len(x) > 1 && (x[0] == "e" || x[0] == "p") {
isBin = true
}
for i, y := range x {
if i == 1 && isBin {
var b []byte
b, err = hex.Dec(y)
if err != nil {
err = errorf.E("e or p tag value not hex: %s", err.Error())
return
}
if len(b) != 32 {
err = errorf.E("e or p tag value with invalid decoded byte length %d", len(b))
return
}
varint.Encode(w, uint64(len(b)))
_, _ = w.Write(b)
} else {
varint.Encode(w, uint64(len(y)))
_, _ = w.Write([]byte(y))
}
}
}
varint.Encode(w, uint64(len(ev.Content)))
_, _ = w.Write([]byte(ev.Content))
_, _ = w.Write(ev.GetSigBytes())
return
return err
}
// UnmarshalRead decodes an event in binary form into an allocated event struct.
//
// NOTE: Event must not be nil or this will panic. Use event.New.
func (ev *E) UnmarshalRead(r io.Reader) (err error) {
if ev == nil {
panic("cannot unmarshal into nil event struct")
}
id := make([]byte, 32)
if _, err = r.Read(id); chk.E(err) {
return
@@ -75,7 +109,8 @@ func (ev *E) UnmarshalRead(r io.Reader) (err error) {
return
}
var t []string
for range nField {
var isBin bool
for i := range nField {
var lenField uint64
if lenField, err = varint.Decode(r); chk.E(err) {
return
@@ -84,6 +119,17 @@ func (ev *E) UnmarshalRead(r io.Reader) (err error) {
if _, err = r.Read(field); chk.E(err) {
return
}
// if it is first field, length 1 and is e or p, the value field must be binary
if i == 0 && len(field) == 1 && (field[0] == 'e' || field[0] == 'p') {
isBin = true
}
if i == 1 && isBin {
// this is a binary value, was an e or p tag key, 32 bytes long, encode value
// field to hex
f := make([]byte, 64)
_ = hex.EncBytes(f, field)
field = f
}
t = append(t, string(field))
}
ev.Tags = append(ev.Tags, t)

44
event/event.go
View File

@@ -22,7 +22,7 @@ type E struct {
Id string `json:"id"`
Pubkey string `json:"pubkey"`
CreatedAt timestamp.Timestamp `json:"created_at"`
Kind int `json:"kind`
Kind int `json:"kind"`
Tags tags.Tags `json:"tags"`
Content string `json:"content"`
Sig string `json:"sig"`
@@ -30,14 +30,45 @@ type E struct {
func New() (ev *E) { return &E{} }
func (ev *E) Marshal() (b []byte, err error) {
if b, err = json.Marshal(ev); chk.E(err) {
func (ev *E) IdBytes() (idBytes []byte, err error) {
if idBytes, err = hex.Dec(ev.Id); chk.E(err) {
return
}
return
}
func (ev *E) PubBytes() (pubBytes []byte, err error) {
if pubBytes, err = hex.Dec(ev.Pubkey); chk.E(err) {
return
}
return
}
func (ev *E) SigBytes() (sigBytes []byte, err error) {
if sigBytes, err = hex.Dec(ev.Sig); chk.E(err) {
return
}
return
}
func (ev *E) Marshal() (b []byte, err error) {
if ev == nil {
panic("cannot marshal a nil event")
}
if b, err = json.Marshal(ev); chk.E(err) {
return
}
// there is a problem with some specific characters here
b = bytes.ReplaceAll(b, []byte("\\u0026"), []byte("&"))
return
}
func (ev *E) Unmarshal(b []byte) (err error) {
if ev == nil {
panic("cannot unmarshal into a nil event")
}
// there is a problem with some specific characters here
b = bytes.ReplaceAll(b, []byte("\\u0026"), []byte("&"))
if err = json.Unmarshal(b, ev); chk.E(err) {
return
}
@@ -48,9 +79,14 @@ func (ev *E) Unmarshal(b []byte) (err error) {
// logging.
func (ev *E) Serialize() (b []byte) {
var err error
if len(ev.Tags) == 1 && len(ev.Tags[0]) == 1 {
ev.Tags = ev.Tags[:0]
}
if b, err = json.Marshal(ev); chk.E(err) {
return
}
// there is a problem with some specific characters here
b = bytes.ReplaceAll(b, []byte("\\u0026"), []byte("&"))
return
}
@@ -80,7 +116,7 @@ func (ev *E) Verify() (valid bool, err error) {
// check that this isn't because of a bogus Id
id := ev.GenIdBytes()
if !bytes.Equal(id, ev.GetIdBytes()) {
log.E.Ln("event Id incorrect")
log.E.F("event Id incorrect\n%s\n%s", ev.Serialize(), ev.ToCanonical(nil))
ev.Id = hex.Enc(id)
err = nil
if valid, err = keys.Verify(ev.GetIdBytes(), ev.GetSigBytes()); chk.E(err) {

16
go.mod
View File

@@ -4,7 +4,9 @@ go 1.24.3
require (
github.com/alexflint/go-arg v1.5.1
github.com/clipperhouse/uax29 v1.14.3
github.com/davecgh/go-spew v1.1.1
github.com/dgraph-io/badger/v4 v4.7.0
github.com/fatih/color v1.18.0
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
github.com/mailru/easyjson v0.9.0
@@ -22,18 +24,32 @@ require (
require (
github.com/BurntSushi/toml v1.5.0 // indirect
github.com/alexflint/go-scalar v1.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/felixge/fgprof v0.9.5 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/pprof v0.0.0-20250501235452-c0086092b71a // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/templexxx/cpu v0.1.1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.35.0 // indirect
go.opentelemetry.io/otel/trace v1.35.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20250530174510-65e920069ea6 // indirect
golang.org/x/mod v0.24.0 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/tools v0.33.0 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

61
go.sum
View File

@@ -1,11 +1,11 @@
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y=
github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8=
github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw=
github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
@@ -15,19 +15,36 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
github.com/clipperhouse/uax29 v1.14.3 h1:pJ0hZWycgsBrJ8SSsvPCrlMTpW8C+fdcA/0mehFDCU0=
github.com/clipperhouse/uax29 v1.14.3/go.mod h1:paNABhygWmmjkg0ROxKQoenJAX4dM9AS8biVkXmAK0c=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y=
github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA=
github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=
github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY=
github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/pprof v0.0.0-20250501235452-c0086092b71a h1:rDA3FfmxwXR+BVKKdz55WwMJ1pD2hJQNW31d+l3mPk4=
@@ -38,10 +55,15 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
@@ -56,6 +78,8 @@ github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -70,28 +94,30 @@ github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b h1:XeDLE6c9mzHpdv3W
github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b/go.mod h1:7rwmCH0wC2fQvNEvPZ3sKXukhyCTyiaZ5VTZMQYpZKQ=
go-simpler.org/env v0.12.0 h1:kt/lBts0J1kjWJAnB740goNdvwNxt5emhYngL0Fzufs=
go-simpler.org/env v0.12.0/go.mod h1:cc/5Md9JCUM7LVLtN0HYjPTDcI3Q8TDaPlNTAlDU+WI=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
golang.org/x/exp v0.0.0-20250530174510-65e920069ea6 h1:gllJVKwONftmCc4KlNbN8o/LvmbxotqQy6zzi6yDQOQ=
golang.org/x/exp v0.0.0-20250530174510-65e920069ea6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/exp/typeparams v0.0.0-20250530174510-65e920069ea6 h1:Gq937g8bNiCnWB/wsoyxuxnfDpAE9cpYo4sLIp9t0LA=
golang.org/x/exp/typeparams v0.0.0-20250530174510-65e920069ea6/go.mod h1:LKZHyeOpPuZcMgxeHjJp4p5yvxrCX1xDvH10zYHhjjQ=
golang.org/x/lint v0.0.0-20241112194109-818c5a804067 h1:adDmSQyFTCiv19j015EGKJBoaa7ElV0Q1Wovb/4G7NA=
golang.org/x/lint v0.0.0-20241112194109-818c5a804067/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -102,14 +128,17 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

1
ints/base10k.txt Normal file
View File

File diff suppressed because one or more lines are too long

20
ints/gen/pregen.go Normal file
View File

@@ -0,0 +1,20 @@
// Package main is a generator for the base10000 (4 digit) encoding of the ints
// library.
package main
import (
"fmt"
"os"
"x.realy.lol/chk"
)
func main() {
fh, err := os.Create("pkg/ints/base10k.txt")
if chk.E(err) {
panic(err)
}
for i := range 10000 {
fmt.Fprintf(fh, "%04d", i)
}
}

134
ints/ints.go Normal file
View File

@@ -0,0 +1,134 @@
// Package ints is an optimised encoder for decimal numbers in ASCII format,
// that simplifies and accelerates encoding and decoding decimal strings. It is
// faster than strconv in part because it uses a base of 10000 and a lookup
// table.
package ints
import (
_ "embed"
"io"
"golang.org/x/exp/constraints"
"x.realy.lol/errorf"
)
// run this to regenerate (pointlessly) the base 10 array of 4 places per entry
//go:generate go run ./gen/.
//go:embed base10k.txt
var base10k []byte
// T is an integer with a fast codec to decimal ASCII.
type T struct {
N uint64
}
func New[V constraints.Integer](n V) *T {
return &T{uint64(n)}
}
// Uint64 returns the int.T as a uint64 (the base type)
func (n *T) Uint64() uint64 { return n.N }
// Int64 returns an int64 from the base number (may cause truncation)
func (n *T) Int64() int64 { return int64(n.N) }
// Uint16 returns an uint16 from the base number (may cause truncation)
func (n *T) Uint16() uint16 { return uint16(n.N) }
var powers = []*T{
{1},
{1_0000},
{1_0000_0000},
{1_0000_0000_0000},
{1_0000_0000_0000_0000},
}
const zero = '0'
const nine = '9'
// Marshal the int.T into a byte string.
func (n *T) Marshal(dst []byte) (b []byte) {
nn := n.N
b = dst
if n.N == 0 {
b = append(b, '0')
return
}
var i int
var trimmed bool
k := len(powers)
for k > 0 {
k--
q := n.N / powers[k].N
if !trimmed && q == 0 {
continue
}
offset := q * 4
bb := base10k[offset : offset+4]
if !trimmed {
for i = range bb {
if bb[i] != '0' {
bb = bb[i:]
trimmed = true
break
}
}
}
b = append(b, bb...)
n.N = n.N - q*powers[k].N
}
n.N = nn
return
}
// Unmarshal reads a string, which must be a positive integer no larger than math.MaxUint64,
// skipping any non-numeric content before it.
//
// Note that leading zeros are not considered valid, but basically no such thing as machine
// generated JSON integers with leading zeroes. Until this is disproven, this is the fastest way
// to read a positive json integer, and a leading zero is decoded as a zero, and the remainder
// returned.
func (n *T) Unmarshal(b []byte) (r []byte, err error) {
if len(b) < 1 {
err = errorf.E("zero length number")
return
}
var sLen int
if b[0] == zero {
r = b[1:]
n.N = 0
return
}
// skip non-number characters
for i, v := range b {
if v >= '0' && v <= '9' {
b = b[i:]
break
}
}
if len(b) == 0 {
err = io.EOF
return
}
// count the digits
for ; sLen < len(b) && b[sLen] >= zero && b[sLen] <= nine && b[sLen] != ','; sLen++ {
}
if sLen == 0 {
err = errorf.E("zero length number")
return
}
if sLen > 20 {
err = errorf.E("too big number for uint64")
return
}
// the length of the string found
r = b[sLen:]
b = b[:sLen]
for _, ch := range b {
ch -= zero
n.N = n.N*10 + uint64(ch)
}
return
}

79
ints/ints_test.go Normal file
View File

@@ -0,0 +1,79 @@
package ints
import (
"math"
"strconv"
"testing"
"lukechampine.com/frand"
"x.realy.lol/chk"
)
func TestMarshalUnmarshal(t *testing.T) {
b := make([]byte, 0, 8)
var rem []byte
var n *T
var err error
for _ = range 10000000 {
n = New(uint64(frand.Intn(math.MaxInt64)))
b = n.Marshal(b)
m := New(0)
if rem, err = m.Unmarshal(b); chk.E(err) {
t.Fatal(err)
}
if n.N != m.N {
t.Fatalf("failed to convert to int64 at %d %s %d", n.N, b, m.N)
}
if len(rem) > 0 {
t.Fatalf("leftover bytes after converting back: '%s'", rem)
}
b = b[:0]
}
}
func BenchmarkByteStringToInt64(bb *testing.B) {
b := make([]byte, 0, 19)
var i int
const nTests = 10000000
testInts := make([]*T, nTests)
for i = range nTests {
testInts[i] = New(frand.Intn(math.MaxInt64))
}
bb.Run("Marshal", func(bb *testing.B) {
bb.ReportAllocs()
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
b = n.Marshal(b)
b = b[:0]
}
})
bb.Run("Itoa", func(bb *testing.B) {
bb.ReportAllocs()
var s string
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
s = strconv.Itoa(int(n.N))
_ = s
}
})
bb.Run("MarshalUnmarshal", func(bb *testing.B) {
bb.ReportAllocs()
m := New(0)
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
b = m.Marshal(b)
_, _ = n.Unmarshal(b)
b = b[:0]
}
})
bb.Run("ItoaAtoi", func(bb *testing.B) {
bb.ReportAllocs()
var s string
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
s = strconv.Itoa(int(n.N))
_, _ = strconv.Atoi(s)
}
})
}

291
kind/kinds.go
View File

@@ -1,135 +1,135 @@
package kind
const (
ProfileMetadata int = 0
TextNote int = 1
RecommendServer int = 2
FollowList int = 3
EncryptedDirectMessage int = 4
Deletion int = 5
Repost int = 6
Reaction int = 7
BadgeAward int = 8
SimpleGroupChatMessage int = 9
SimpleGroupThreadedReply int = 10
SimpleGroupThread int = 11
SimpleGroupReply int = 12
Seal int = 13
DirectMessage int = 14
GenericRepost int = 16
ReactionToWebsite int = 17
ChannelCreation int = 40
ChannelMetadata int = 41
ChannelMessage int = 42
ChannelHideMessage int = 43
ChannelMuteUser int = 44
Chess int = 64
MergeRequests int = 818
Bid int = 1021
BidConfirmation int = 1022
OpenTimestamps int = 1040
GiftWrap int = 1059
FileMetadata int = 1063
LiveChatMessage int = 1311
Patch int = 1617
Issue int = 1621
Reply int = 1622
StatusOpen int = 1630
StatusApplied int = 1631
StatusClosed int = 1632
StatusDraft int = 1633
ProblemTracker int = 1971
Reporting int = 1984
Label int = 1985
RelayReviews int = 1986
AIEmbeddings int = 1987
Torrent int = 2003
TorrentComment int = 2004
CoinjoinPool int = 2022
CommunityPostApproval int = 4550
JobFeedback int = 7000
SimpleGroupPutUser int = 9000
SimpleGroupRemoveUser int = 9001
SimpleGroupEditMetadata int = 9002
SimpleGroupDeleteEvent int = 9005
SimpleGroupCreateGroup int = 9007
SimpleGroupDeleteGroup int = 9008
SimpleGroupCreateInvite int = 9009
SimpleGroupJoinRequest int = 9021
SimpleGroupLeaveRequest int = 9022
ZapGoal int = 9041
TidalLogin int = 9467
ZapRequest int = 9734
Zap int = 9735
Highlights int = 9802
MuteList int = 10000
PinList int = 10001
RelayListMetadata int = 10002
BookmarkList int = 10003
CommunityList int = 10004
PublicChatList int = 10005
BlockedRelayList int = 10006
SearchRelayList int = 10007
SimpleGroupList int = 10009
InterestList int = 10015
EmojiList int = 10030
DMRelayList int = 10050
UserServerList int = 10063
FileStorageServerList int = 10096
GoodWikiAuthorList int = 10101
GoodWikiRelayList int = 10102
NWCWalletInfo int = 13194
LightningPubRPC int = 21000
ClientAuthentication int = 22242
NWCWalletRequest int = 23194
NWCWalletResponse int = 23195
NostrConnect int = 24133
Blobs int = 24242
HTTPAuth int = 27235
CategorizedPeopleList int = 30000
CategorizedBookmarksList int = 30001
RelaySets int = 30002
BookmarkSets int = 30003
CuratedSets int = 30004
CuratedVideoSets int = 30005
MuteSets int = 30007
ProfileBadges int = 30008
BadgeDefinition int = 30009
InterestSets int = 30015
StallDefinition int = 30017
ProductDefinition int = 30018
MarketplaceUI int = 30019
ProductSoldAsAuction int = 30020
Article int = 30023
DraftArticle int = 30024
EmojiSets int = 30030
ModularArticleHeader int = 30040
ModularArticleContent int = 30041
ReleaseArtifactSets int = 30063
ApplicationSpecificData int = 30078
LiveEvent int = 30311
UserStatuses int = 30315
ClassifiedListing int = 30402
DraftClassifiedListing int = 30403
RepositoryAnnouncement int = 30617
RepositoryState int = 30618
SimpleGroupMetadata int = 39000
SimpleGroupAdmins int = 39001
SimpleGroupMembers int = 39002
SimpleGroupRoles int = 39003
WikiArticle int = 30818
Redirects int = 30819
Feed int = 31890
DateCalendarEvent int = 31922
TimeCalendarEvent int = 31923
Calendar int = 31924
CalendarEventRSVP int = 31925
HandlerRecommendation int = 31989
HandlerInformation int = 31990
VideoEvent int = 34235
ShortVideoEvent int = 34236
VideoViewEvent int = 34237
CommunityDefinition int = 34550
ProfileMetadata = 0
TextNote = 1
RecommendServer = 2
FollowList = 3
EncryptedDirectMessage = 4
Deletion = 5
Repost = 6
Reaction = 7
BadgeAward = 8
SimpleGroupChatMessage = 9
SimpleGroupThreadedReply = 10
SimpleGroupThread = 11
SimpleGroupReply = 12
Seal = 13
DirectMessage = 14
GenericRepost = 16
ReactionToWebsite = 17
ChannelCreation = 40
ChannelMetadata = 41
ChannelMessage = 42
ChannelHideMessage = 43
ChannelMuteUser = 44
Chess = 64
MergeRequests = 818
Bid = 1021
BidConfirmation = 1022
OpenTimestamps = 1040
GiftWrap = 1059
FileMetadata = 1063
LiveChatMessage = 1311
Patch = 1617
Issue = 1621
Reply = 1622
StatusOpen = 1630
StatusApplied = 1631
StatusClosed = 1632
StatusDraft = 1633
ProblemTracker = 1971
Reporting = 1984
Label = 1985
RelayReviews = 1986
AIEmbeddings = 1987
Torrent = 2003
TorrentComment = 2004
CoinjoinPool = 2022
CommunityPostApproval = 4550
JobFeedback = 7000
SimpleGroupPutUser = 9000
SimpleGroupRemoveUser = 9001
SimpleGroupEditMetadata = 9002
SimpleGroupDeleteEvent = 9005
SimpleGroupCreateGroup = 9007
SimpleGroupDeleteGroup = 9008
SimpleGroupCreateInvite = 9009
SimpleGroupJoinRequest = 9021
SimpleGroupLeaveRequest = 9022
ZapGoal = 9041
TidalLogin = 9467
ZapRequest = 9734
Zap = 9735
Highlights = 9802
MuteList = 10000
PinList = 10001
RelayListMetadata = 10002
BookmarkList = 10003
CommunityList = 10004
PublicChatList = 10005
BlockedRelayList = 10006
SearchRelayList = 10007
SimpleGroupList = 10009
InterestList = 10015
EmojiList = 10030
DMRelayList = 10050
UserServerList = 10063
FileStorageServerList = 10096
GoodWikiAuthorList = 10101
GoodWikiRelayList = 10102
NWCWalletInfo = 13194
LightningPubRPC = 21000
ClientAuthentication = 22242
NWCWalletRequest = 23194
NWCWalletResponse = 23195
NostrConnect = 24133
Blobs = 24242
HTTPAuth = 27235
CategorizedPeopleList = 30000
CategorizedBookmarksList = 30001
RelaySets = 30002
BookmarkSets = 30003
CuratedSets = 30004
CuratedVideoSets = 30005
MuteSets = 30007
ProfileBadges = 30008
BadgeDefinition = 30009
InterestSets = 30015
StallDefinition = 30017
ProductDefinition = 30018
MarketplaceUI = 30019
ProductSoldAsAuction = 30020
Article = 30023
DraftArticle = 30024
EmojiSets = 30030
ModularArticleHeader = 30040
ModularArticleContent = 30041
ReleaseArtifactSets = 30063
ApplicationSpecificData = 30078
LiveEvent = 30311
UserStatuses = 30315
ClassifiedListing = 30402
DraftClassifiedListing = 30403
RepositoryAnnouncement = 30617
RepositoryState = 30618
SimpleGroupMetadata = 39000
SimpleGroupAdmins = 39001
SimpleGroupMembers = 39002
SimpleGroupRoles = 39003
WikiArticle = 30818
Redirects = 30819
Feed = 31890
DateCalendarEvent = 31922
TimeCalendarEvent = 31923
Calendar = 31924
CalendarEventRSVP = 31925
HandlerRecommendation = 31989
HandlerInformation = 31990
VideoEvent = 34235
ShortVideoEvent = 34236
VideoViewEvent = 34237
CommunityDefinition = 34550
)
func IsRegularKind(kind int) bool {
@@ -147,3 +147,36 @@ func IsEphemeralKind(kind int) bool {
func IsAddressableKind(kind int) bool {
return 30000 <= kind && kind < 40000
}
var Text = []int{
ProfileMetadata,
TextNote,
Article,
SimpleGroupThread,
Reply,
Repost,
Issue,
Reply,
MergeRequests,
WikiArticle,
Issue,
StatusOpen,
StatusApplied,
StatusClosed,
StatusDraft,
Torrent,
TorrentComment,
DateCalendarEvent,
TimeCalendarEvent,
Calendar,
CalendarEventRSVP,
}
func IsText(ki int) bool {
for _, v := range Text {
if v == ki {
return true
}
}
return false
}

81
tags/tags.go
View File

@@ -7,7 +7,10 @@ import (
"slices"
"strings"
"x.realy.lol/chk"
"x.realy.lol/helpers"
"x.realy.lol/hex"
"x.realy.lol/ints"
"x.realy.lol/normalize"
)
@@ -106,6 +109,18 @@ func (tags Tags) GetAll(tagPrefix []string) Tags {
return result
}
func (tags Tags) GetAllExactKeys(key string) Tags {
result := make(Tags, 0, len(tags))
for _, v := range tags {
if v.StartsWith([]string{key}) {
if v.Key() == key {
result = append(result, v)
}
}
}
return result
}
// All returns an iterator for all the tags that match the prefix, see [Tag.StartsWith]
func (tags Tags) All(tagPrefix []string) iter.Seq2[int, Tag] {
return func(yield func(int, Tag) bool) {
@@ -131,17 +146,18 @@ func (tags Tags) FilterOut(tagPrefix []string) Tags {
}
// FilterOutInPlace removes all tags that match the prefix, but potentially reorders the tags in unpredictable ways, see [Tag.StartsWith]
func (tags *Tags) FilterOutInPlace(tagPrefix []string) {
for i := 0; i < len(*tags); i++ {
tag := (*tags)[i]
func (tags Tags) FilterOutInPlace(tagPrefix []string) (t Tags) {
for i := 0; i < len(tags); i++ {
tag := (tags)[i]
if tag.StartsWith(tagPrefix) {
// remove this by swapping the last tag into this place
last := len(*tags) - 1
(*tags)[i] = (*tags)[last]
*tags = (*tags)[0:last]
last := len(tags) - 1
(tags)[i] = (tags)[last]
tags = (tags)[0:last]
i-- // this is so we can match this just swapped item in the next iteration
}
}
return tags
}
// AppendUnique appends a tag if it doesn't exist yet, otherwise does nothing.
@@ -158,7 +174,7 @@ func (tags Tags) AppendUnique(tag Tag) Tags {
return tags
}
func (t *Tags) Scan(src any) error {
func (tags Tags) Scan(src any) (t Tags, err error) {
var jtags []byte
switch v := src.(type) {
@@ -167,11 +183,11 @@ func (t *Tags) Scan(src any) error {
case string:
jtags = []byte(v)
default:
return errors.New("couldn't scan tags, it's not a json string")
err = errors.New("couldn't scan tags, it's not a json string")
return
}
json.Unmarshal(jtags, &t)
return nil
err = json.Unmarshal(jtags, &tags)
return
}
func (tags Tags) ContainsAny(tagName string, values []string) bool {
@@ -218,3 +234,46 @@ func (tags Tags) marshalTo(dst []byte) []byte {
dst = append(dst, ']')
return dst
}
type Tag_a struct {
Kind int
Pubkey []byte
Ident string
}
func (tags Tags) Get_a_Tags() (atags []Tag_a) {
a := tags.GetAll([]string{"a"})
var err error
if len(a) > 0 {
for _, v := range a {
if v[0] == "a" && len(v) > 1 {
// try to split it
parts := strings.Split(v[1], ":")
// there must be a kind first
ki := ints.New(0)
if _, err = ki.Unmarshal([]byte(parts[0])); chk.E(err) {
continue
}
atag := Tag_a{
Kind: int(ki.Uint16()),
}
if len(parts) < 2 {
continue
}
// next must be a pubkey
var pk []byte
if pk, err = hex.Dec(parts[1]); chk.E(err) {
continue
}
atag.Pubkey = pk
// there possibly can be nothing after this
if len(parts) >= 3 {
// third part is the identifier (d tag)
atag.Ident = parts[2]
}
atags = append(atags, atag)
}
}
}
return
}

12
units/units.go Normal file
View File

@@ -0,0 +1,12 @@
// Package units is a convenient set of names designating data sizes in bytes
// using common ISO names (base 10).
package units
const (
Kilobyte = 1000
Kb = Kilobyte
Megabyte = Kilobyte * Kilobyte
Mb = Megabyte
Gigabyte = Megabyte * Kilobyte
Gb = Gigabyte
)