ackchually generates events

go.mod

@@ -8,11 +8,14 @@ require (
 	go.uber.org/atomic v1.11.0
 	golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3
 	lukechampine.com/frand v1.5.1
+	realy.lol v1.7.13
 )
 
 require (
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/stretchr/testify v1.10.0 // indirect
+	github.com/templexxx/cpu v0.1.1 // indirect
+	github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b // indirect
 	golang.org/x/sys v0.30.0 // indirect
 )

go.sum

@@ -2,6 +2,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
+github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -10,18 +12,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/templexxx/cpu v0.0.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk=
+github.com/templexxx/cpu v0.1.1 h1:isxHaxBXpYFWnk2DReuKkigaZyrjs2+9ypIdGP4h+HI=
+github.com/templexxx/cpu v0.1.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk=
+github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b h1:XeDLE6c9mzHpdv3Wb1+pWBaWv/BlHK0ZYIu/KaL6eHg=
+github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b/go.mod h1:7rwmCH0wC2fQvNEvPZ3sKXukhyCTyiaZ5VTZMQYpZKQ=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc=
-golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 lukechampine.com/frand v1.5.1 h1:fg0eRtdmGFIxhP5zQJzM1lFDbD6CUfu/f+7WgAZd5/w=
 lukechampine.com/frand v1.5.1/go.mod h1:4VstaWc2plN4Mjr10chUD46RAVGWhpkZ5Nja8+Azp0Q=
+realy.lol v1.7.13 h1:+7kIa+RFmvdP23DRjj1GEe7+F7cmyl/xuII8QMwe7nM=
+realy.lol v1.7.13/go.mod h1:qtk9aklmo7dpX+uSj20ol4utmh3ldWXQOpyzH4dcRG8=

pkg/content/content.go

@@ -20,7 +20,7 @@ func (c *C) Marshal(d []byte) (r []byte, err error) {
 	r = append(r, '\n')
 	// log.I.S(r)
 	r = append(r, c.Content...)
-	r = append(r, '\n')
+	// r = append(r, '\n')
 	// log.I.S(r)
 	return
 }
@@ -63,4 +63,4 @@ func (c *C) Unmarshal(d []byte) (r []byte, err error) {
 	}
 	r = r[1:]
 	return
-}
+}

pkg/content/content_test.go

@@ -5,6 +5,8 @@ import (
 	"crypto/rand"
 	mrand "math/rand"
 	"testing"
+
+	"protocol.realy.lol/pkg/separator"
 )
 
 func TestC_Marshal_Unmarshal(t *testing.T) {
@@ -19,6 +21,7 @@ func TestC_Marshal_Unmarshal(t *testing.T) {
 	if res, err = c1.Marshal(nil); chk.E(err) {
 		t.Fatal(err)
 	}
+	res = separator.Add(res)
 	c2 := new(C)
 	var rem []byte
 	if rem, err = c2.Unmarshal(res); chk.E(err) {
@@ -32,4 +35,4 @@ func TestC_Marshal_Unmarshal(t *testing.T) {
 		log.I.S(rem)
 		t.Fatalf("unexpected remaining bytes: '%0x'", rem)
 	}
-}
+}

pkg/decimal/decimal.go

@@ -70,6 +70,7 @@ func (n *T) Marshal(d []byte) (r []byte, err error) {
 		r = append(r, bb...)
 		n.N = n.N - q*powers[k].N
 	}
+	// r = append(r, '\n')
 	n.N = nn
 	return
 }
@@ -111,4 +112,4 @@ func (n *T) Unmarshal(d []byte) (r []byte, err error) {
 		n.N = n.N*10 + uint64(ch)
 	}
 	return
-}
+}

pkg/event/event.go

@@ -1,21 +1,27 @@
 package event
 
 import (
+	"realy.lol/sha256"
+	"realy.lol/signer"
+
 	"protocol.realy.lol/pkg/content"
 	"protocol.realy.lol/pkg/decimal"
-	"protocol.realy.lol/pkg/event/types"
 	"protocol.realy.lol/pkg/pubkey"
+	"protocol.realy.lol/pkg/separator"
 	"protocol.realy.lol/pkg/signature"
 	"protocol.realy.lol/pkg/tags"
+	"protocol.realy.lol/pkg/types"
 )
 
 type E struct {
+	id        []byte
 	Type      *types.T
 	Pubkey    *pubkey.P
 	Timestamp *decimal.T
 	Tags      *tags.T
 	Content   *content.C
 	Signature *signature.S
+	encoded   []byte
 }
 
 // New creates a new event with some typical data already filled. This should be
@@ -44,30 +50,91 @@ func New(pk []byte, typ string) (ev *E, err error) {
 	return
 }
 
-func (e *E) Marshal(d []byte) (r []byte, err error) {
+// Invalidate empties the existing encoded cache of the event. This needs to be
+// called in case of mutating its fields. It also nils the signature.
+func (e *E) Invalidate() { e.encoded = e.encoded[:0]; e.Signature = nil; e.id = nil }
+
+func (e *E) Sign(s signer.I) (err error) {
+	var h []byte
+	if h, err = e.Hash(); chk.E(err) {
+		return
+	}
+	var sig []byte
+	if sig, err = s.Sign(h); chk.E(err) {
+		return
+	}
+	if e.Signature, err = signature.New(sig); chk.E(err) {
+		return
+	}
+	return
+}
+
+func (e *E) Encode(d []byte) (r []byte, err error) {
 	r = d
-	if r, err = e.Type.Marshal(d); chk.E(err) {
+	if e.Type == nil {
+		err = errorf.E("type is not defined for event")
 		return
 	}
-	if r, err = e.Pubkey.Marshal(d); chk.E(err) {
+	if r, err = e.Type.Marshal(r); chk.E(err) {
 		return
 	}
-	if r, err = e.Timestamp.Marshal(d); chk.E(err) {
+	r = separator.Add(r)
+	if e.Pubkey == nil {
+		err = errorf.E("pubkey is not defined for event")
 		return
 	}
-	if r, err = e.Tags.Marshal(d); chk.E(err) {
+	// log.I.S(r)
+	if r, err = e.Pubkey.Marshal(r); chk.E(err) {
 		return
 	}
-	if r, err = e.Content.Marshal(d); chk.E(err) {
+	r = separator.Add(r)
+	if e.Timestamp == nil {
+		err = errorf.E("timestamp is not defined for event")
 		return
 	}
-	if r, err = e.Signature.Marshal(d); chk.E(err) {
+	if r, err = e.Timestamp.Marshal(r); chk.E(err) {
 		return
 	}
+	r = separator.Add(r)
+	if r, err = e.Tags.Marshal(r); chk.E(err) {
+		return
+	}
+	if e.Content != nil {
+		if r, err = e.Content.Marshal(r); chk.E(err) {
+			return
+		}
+		r = separator.Add(r)
+	}
+	e.encoded = r
+	return
+}
+
+func (e *E) Hash() (h []byte, err error) {
+	var b []byte
+	if e.encoded == nil {
+		if e.encoded, err = e.Encode(nil); chk.E(err) {
+			return
+		}
+		b = e.encoded
+	}
+	hh := sha256.Sum256(b)
+	h = hh[:]
+	e.id = h
+	return
+}
+
+func (e *E) Marshal(d []byte) (r []byte, err error) {
+	if r, err = e.Encode(d); chk.E(err) {
+		return
+	}
+	if r, err = e.Signature.Marshal(r); chk.E(err) {
+		return
+	}
+	r = separator.Add(r)
 	return
 }
 
 func (e *E) Unmarshal(data []byte) (r []byte, err error) {
 
 	return
-}
+}

pkg/event/event_test.go

@@ -1,9 +1,205 @@
 package event
 
 import (
+	"encoding/binary"
+	mrand "math/rand"
+	"math/rand/v2"
 	"testing"
+	"time"
+
+	"lukechampine.com/frand"
+	"realy.lol/signer"
+
+	"protocol.realy.lol/pkg/content"
+	"protocol.realy.lol/pkg/decimal"
+	"protocol.realy.lol/pkg/id"
+	"protocol.realy.lol/pkg/pubkey"
+	"protocol.realy.lol/pkg/tag"
+	"protocol.realy.lol/pkg/tags"
+	"protocol.realy.lol/pkg/types"
+
+	"realy.lol/p256k"
 )
 
-func TestE_Marshal_Unmarshal(t *testing.T) {
+const seed = 0
 
+func GenerateFake32Bytes(rng *rand.Rand) (fake []byte) {
+	fake = make([]byte, 32)
+	for i := range 4 {
+		n := rng.Uint64()
+		binary.LittleEndian.PutUint64(fake[i*8:i*8+8], n)
+	}
+	return
 }
+
+var Hashtags, _ = tag.New(
+	"halsey",
+	"$DIAM",
+	"Trevor Lawrence",
+	"#AEWCEO",
+	"Reuters",
+	"Linda McMahon",
+	"Bolton",
+	"Raining in Houston",
+	"#SwiftDay",
+	"Munich",
+	"NATO",
+	"#thursdayvibes",
+	"Good Thursday",
+	"$SEA",
+	"#AEWGrandSlam",
+	"Brian Steele",
+	"#GalentinesDay",
+	"Bregman",
+	"Afghan",
+	"The Accountant 2",
+	"Happy Friday Eve",
+	"TLaw",
+	"Red Sox",
+	"Large Scale Social Deception",
+	"2024 BMW",
+	"Onew",
+	"Secretary of Education",
+	"$HIMS",
+	"Core PPI",
+	"Avowed",
+	"Kemp",
+	"Angel's Venture",
+	"YouTube TV",
+	"Bri Bri",
+	"Teslas",
+	"Thirsty Thursday",
+	"matz",
+	"Jack the Ripper",
+	"Paramount",
+	"Megan Boswell",
+	"Zeldin",
+	"Zelensky",
+	"Censure",
+	"Sheldon Whitehouse",
+	"Arenado",
+	"Parasite Class",
+	"Kennedy Center",
+	"I Love Jesus",
+	"James Cook",
+)
+
+func GenerateContent(rng *rand.Rand, l int) (c *content.C) {
+	c = &content.C{}
+
+	return
+}
+
+const lorem = `Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+
+Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. 
+
+Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.`
+
+func GenerateTags(rng *rand.Rand, n int) (t *tags.T, err error) {
+	nE, nP, nH := rng.IntN(n)+1, rng.IntN(n)+1, rng.IntN(n)+1
+	var tt []*tag.T
+	k := tag.List.GetElementBytes(tag.KeyEvent)
+	for range nE {
+		var tg *tag.T
+		v := GenerateFake32Bytes(rng)
+		var e *id.T
+		if e, err = id.New(v); chk.E(err) {
+			return
+		}
+		var b []byte
+		if b, err = e.Marshal(b); chk.E(err) {
+			return
+		}
+		if tg, err = tag.New(k, b, []byte("root")); chk.E(err) {
+			return
+		}
+		tt = append(tt, tg)
+	}
+	k = tag.List.GetElementBytes(tag.KeyPubkey)
+	for range nP {
+		var tg *tag.T
+		v := GenerateFake32Bytes(rng)
+		var p *pubkey.P
+		if p, err = pubkey.New(v); chk.E(err) {
+			return
+		}
+		var b []byte
+		if b, err = p.Marshal(b); chk.E(err) {
+			return
+		}
+		if tg, err = tag.New(k, b); chk.E(err) {
+			return
+		}
+		tt = append(tt, tg)
+	}
+	k = tag.List.GetElementBytes(tag.KeyHashtag)
+	for range nH {
+		var tg *tag.T
+		v := Hashtags.GetElementBytes(rng.IntN(Hashtags.Len() - 1))
+		// v = bytes.ReplaceAll(v, []byte{';'}, []byte{'_'})
+		// v = bytes.ReplaceAll(v, []byte{':'}, []byte{'-'})
+		// log.I.S(v)
+		if tg, err = tag.New(k, v); chk.E(err) {
+			return
+		}
+		tt = append(tt, tg)
+	}
+	t = tags.New(tt...)
+	return
+}
+
+func GenerateEvent(sign signer.I) (ev *E, err error) {
+	s2 := rand.NewPCG(uint64(time.Now().UnixNano()), seed)
+	rng := rand.New(s2)
+	sign = new(p256k.Signer)
+	if err = sign.Generate(); chk.E(err) {
+		return
+	}
+	var pk *pubkey.P
+	if pk, err = pubkey.New(sign.Pub()); chk.E(err) {
+		return
+	}
+	var t *tags.T
+	if t, err = GenerateTags(rng, 3+1); chk.E(err) {
+		return
+	}
+	cont := make([]byte, mrand.Intn(100)+25)
+	_, err = frand.Read(cont)
+
+	ev = &E{
+		Type:      types.New("note/adoc"),
+		Pubkey:    pk,
+		Timestamp: decimal.New(time.Now().Unix()),
+		Tags:      t,
+		Content:   &content.C{Content: []byte(lorem)},
+	}
+	if err = ev.Sign(sign); chk.E(err) {
+		return
+	}
+	return
+}
+
+func TestE_Marshal_Unmarshal(t *testing.T) {
+	var ev *E
+	var err error
+	var b1, b2 []byte
+	sign := &p256k.Signer{}
+	if err = sign.Generate(); chk.E(err) {
+		t.Fatal(err)
+	}
+	for range 10 {
+		if ev, err = GenerateEvent(sign); chk.E(err) {
+			t.Fatal(err)
+		}
+		if b1, err = ev.Marshal(b1); chk.E(err) {
+			t.Fatal(err)
+		}
+		log.I.F("\n```\n%s```\n", b1)
+		log.I.S(ev)
+		b1 = b1[:0]
+		_ = b2
+	}
+}

pkg/id/id.go

@@ -9,19 +9,19 @@ import (
 
 const Len = 43
 
-type P struct{ b []byte }
+type T struct{ b []byte }
 
-func New(id []byte) (p *P, err error) {
+func New(id []byte) (p *T, err error) {
 	if len(id) != ed25519.PublicKeySize {
 		err = errorf.E("invalid public key size: %d; require %d",
 			len(id), ed25519.PublicKeySize)
 		return
 	}
-	p = &P{id}
+	p = &T{id}
 	return
 }
 
-func (p *P) Marshal(d []byte) (r []byte, err error) {
+func (p *T) Marshal(d []byte) (r []byte, err error) {
 	r = d
 	if p == nil || p.b == nil || len(p.b) == 0 {
 		err = errorf.E("nil/zero length pubkey")
@@ -40,11 +40,12 @@ func (p *P) Marshal(d []byte) (r []byte, err error) {
 	if err = w.Close(); chk.E(err) {
 		return
 	}
-	r = append(buf.Bytes(), '\n')
+	r = append(r, buf.Bytes()...)
+	// r = append(buf.Bytes(), '\n')
 	return
 }
 
-func (p *P) Unmarshal(data []byte) (r []byte, err error) {
+func (p *T) Unmarshal(data []byte) (r []byte, err error) {
 	r = data
 	if p == nil {
 		err = errorf.E("can't unmarshal into nil types.T")
@@ -71,4 +72,4 @@ func (p *P) Unmarshal(data []byte) (r []byte, err error) {
 	}
 	err = io.EOF
 	return
-}
+}

pkg/id/id_test.go

@@ -5,6 +5,8 @@ import (
 	"crypto/ed25519"
 	"crypto/rand"
 	"testing"
+
+	"protocol.realy.lol/pkg/separator"
 )
 
 func TestT_Marshal_Unmarshal(t *testing.T) {
@@ -14,7 +16,7 @@ func TestT_Marshal_Unmarshal(t *testing.T) {
 		if _, err = rand.Read(pk); chk.E(err) {
 			t.Fatal(err)
 		}
-		var p *P
+		var p *T
 		if p, err = New(pk); chk.E(err) {
 			t.Fatal(err)
 		}
@@ -22,7 +24,8 @@ func TestT_Marshal_Unmarshal(t *testing.T) {
 		if o, err = p.Marshal(nil); chk.E(err) {
 			t.Fatal(err)
 		}
-		p2 := &P{}
+		o = separator.Add(o)
+		p2 := &T{}
 		var rem []byte
 		if rem, err = p2.Unmarshal(o); chk.E(err) {
 			t.Fatal(err)
@@ -34,4 +37,4 @@ func TestT_Marshal_Unmarshal(t *testing.T) {
 			t.Fatal("public key did not encode/decode faithfully")
 		}
 	}
-}
+}

pkg/pubkey/pubkey.go

@@ -32,7 +32,7 @@ func (p *P) Marshal(d []byte) (r []byte, err error) {
 			len(p.PublicKey), ed25519.PublicKeySize, p.PublicKey)
 		return
 	}
-	buf := bytes.NewBuffer(r)
+	buf := new(bytes.Buffer)
 	w := base64.NewEncoder(base64.RawURLEncoding, buf)
 	if _, err = w.Write(p.PublicKey); chk.E(err) {
 		return
@@ -40,7 +40,9 @@ func (p *P) Marshal(d []byte) (r []byte, err error) {
 	if err = w.Close(); chk.E(err) {
 		return
 	}
-	r = append(buf.Bytes(), '\n')
+	// log.I.S(buf.Bytes())
+	r = append(r, buf.Bytes()...)
+	// r = append(buf.Bytes(), '\n')
 	return
 }
 
@@ -71,4 +73,4 @@ func (p *P) Unmarshal(d []byte) (r []byte, err error) {
 	}
 	err = io.EOF
 	return
-}
+}

pkg/pubkey/pubkey_test.go

@@ -5,6 +5,8 @@ import (
 	"crypto/ed25519"
 	"crypto/rand"
 	"testing"
+
+	"protocol.realy.lol/pkg/separator"
 )
 
 func TestP_Marshal_Unmarshal(t *testing.T) {
@@ -22,6 +24,8 @@ func TestP_Marshal_Unmarshal(t *testing.T) {
 		if o, err = p.Marshal(nil); chk.E(err) {
 			t.Fatal(err)
 		}
+		o = separator.Add(o)
+		log.I.S(o)
 		p2 := &P{}
 		var rem []byte
 		if rem, err = p2.Unmarshal(o); chk.E(err) {
@@ -34,4 +38,4 @@ func TestP_Marshal_Unmarshal(t *testing.T) {
 			t.Fatal("public key did not encode/decode faithfully")
 		}
 	}
-}
+}

pkg/separator/separator.go

@@ -0,0 +1,3 @@
+package separator
+
+func Add(dst []byte) (r []byte) { r = append(dst, '\n'); return }

pkg/signature/signature.go

@@ -40,7 +40,7 @@ func (p *S) Marshal(d []byte) (r []byte, err error) {
 			len(p.Signature), ed25519.SignatureSize, p.Signature)
 		return
 	}
-	buf := bytes.NewBuffer(r)
+	buf := new(bytes.Buffer)
 	w := base64.NewEncoder(base64.RawURLEncoding, buf)
 	if _, err = w.Write(p.Signature); chk.E(err) {
 		return
@@ -48,7 +48,8 @@ func (p *S) Marshal(d []byte) (r []byte, err error) {
 	if err = w.Close(); chk.E(err) {
 		return
 	}
-	r = append(buf.Bytes(), '\n')
+	r = append(r, buf.Bytes()...)
+	// r = append(buf.Bytes(), '\n')
 	return
 }
 
@@ -79,4 +80,4 @@ func (p *S) Unmarshal(d []byte) (r []byte, err error) {
 	}
 	err = io.EOF
 	return
-}
+}

pkg/signature/signature_test.go

@@ -5,6 +5,8 @@ import (
 	"crypto/ed25519"
 	"crypto/rand"
 	"testing"
+
+	"protocol.realy.lol/pkg/separator"
 )
 
 func TestS_Marshal_Unmarshal(t *testing.T) {
@@ -22,6 +24,7 @@ func TestS_Marshal_Unmarshal(t *testing.T) {
 		if o, err = s.Marshal(nil); chk.E(err) {
 			t.Fatal(err)
 		}
+		o = separator.Add(o)
 		p2 := &S{}
 		var rem []byte
 		if rem, err = p2.Unmarshal(o); chk.E(err) {
@@ -35,4 +38,4 @@ func TestS_Marshal_Unmarshal(t *testing.T) {
 		}
 
 	}
-}
+}

pkg/tag/common.go

@@ -0,0 +1,17 @@
+package tag
+
+const (
+	KeyEvent = iota
+	KeyPubkey
+	KeyHashtag
+)
+
+var List, _ = New(
+	// event is a reference to an event, the value is an Event Id
+	"event",
+	// pubkey is a reference to a public key, the value is a pubkey.P
+	"pubkey",
+	// hashtag is a string that can be searched by a hashtag filter tag
+	"hashtag",
+	// ... can many more things be in here for purposes
+)

pkg/tag/tag.go

@@ -27,6 +27,7 @@ func New[V ~[]byte | ~string](v ...V) (t *T, err error) {
 	t.fields = append(t.fields, k)
 	for i, val := range v {
 		var b []byte
+		// log.I.S(val)
 		if b, err = ValidateField(val, i); chk.E(err) {
 			return
 		}
@@ -35,6 +36,29 @@ func New[V ~[]byte | ~string](v ...V) (t *T, err error) {
 	return
 }
 
+func (t *T) Len() int           { return len(t.fields) }
+func (t *T) Less(i, j int) bool { return bytes.Compare(t.fields[i], t.fields[j]) < 0 }
+func (t *T) Swap(i, j int)      { t.fields[i], t.fields[j] = t.fields[j], t.fields[i] }
+
+func (t *T) GetElementBytes(i int) (s []byte) {
+	if i >= len(t.fields) {
+		// return empty string if not found
+		return
+	}
+	return t.fields[i]
+}
+
+func (t *T) GetElementString(i int) (s string) {
+	return string(t.GetElementBytes(i))
+}
+
+func (t *T) GetStringSlice() (s []string) {
+	for _, v := range t.fields {
+		s = append(s, string(v))
+	}
+	return
+}
+
 // ValidateKey checks that the key is valid. Keys must be the same most language symbols:
 //
 // - first character is alphabetic [a-zA-Z]
@@ -132,4 +156,4 @@ func (t *T) Unmarshal(d []byte) (r []byte, err error) {
 		}
 	}
 	return
-}
+}

pkg/tags/tags.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"fmt"
 
+	"protocol.realy.lol/pkg/decimal"
+	"protocol.realy.lol/pkg/separator"
 	"protocol.realy.lol/pkg/tag"
 )
 
-const Sentinel = "tags:\n"
+const Sentinel = "tags:"
 
 var SentinelBytes = []byte(Sentinel)
 
@@ -17,15 +19,24 @@ type T struct{ tags }
 
 func New(v ...*tag.T) *T { return &T{tags: v} }
 
-func (t *T) Marshal(dst []byte) (result []byte, err error) {
+func (t *T) Marshal(dst []byte) (r []byte, err error) {
-	result = dst
+	r = dst
-	result = append(result, Sentinel...)
+	r = append(r, Sentinel...)
-	for _, tt := range t.tags {
+	var l int
-		if result, err = tt.Marshal(result); chk.E(err) {
+	if t != nil {
-			return
+		l = len(t.tags)
+	}
+	if r, err = decimal.New(l).Marshal(r); chk.E(err) {
+		return
+	}
+	r = separator.Add(r)
+	if t != nil {
+		for _, tt := range t.tags {
+			if r, err = tt.Marshal(r); chk.E(err) {
+				return
+			}
 		}
 	}
-	result = append(result, '\n')
 	return
 }
 
@@ -34,23 +45,26 @@ func (t *T) Unmarshal(data []byte) (rem []byte, err error) {
 		err = fmt.Errorf("bytes too short to contain tags")
 		return
 	}
-	var dat []byte
+	var d []byte
 	if bytes.Equal(data[:len(Sentinel)], SentinelBytes) {
-		dat = data[len(Sentinel):]
+		d = data[len(Sentinel):]
 	}
-	if len(dat) < 1 {
+	l := decimal.New(0)
+	if d, err = l.Unmarshal(d); chk.E(err) {
 		return
 	}
-	for len(dat) > 0 {
+	// and then there must be a newline
-		if len(dat) == 1 && dat[0] == '\n' {
+	if d[0] != '\n' {
-			break
+		err = errorf.E("must be newline after content:<length>:\n%n", d)
-		}
+		return
-		// log.I.S(dat)
+	}
+	d = d[1:]
+	for range l.N {
 		tt := new(tag.T)
-		if dat, err = tt.Unmarshal(dat); chk.E(err) {
+		if d, err = tt.Unmarshal(d); chk.E(err) {
 			return
 		}
 		t.tags = append(t.tags, tt)
 	}
 	return
-}
+}

pkg/tags/tags_test.go

@@ -9,9 +9,9 @@ import (
 
 func TestT_Marshal_Unmarshal(t *testing.T) {
 	var tegs = [][]string{
-		{"reply", "e:l_T9Of4ru-PLGUxxvw3SfZH0e6XW11VYy8ZSgbcsD9Y", "realy.example.com/repo1"},
+		{"reply", "l_T9Of4ru-PLGUxxvw3SfZH0e6XW11VYy8ZSgbcsD9Y", "realy.example.com/repo1"},
-		{"root", "e:l_T9Of4ru-PLGUxxvw3SfZH0e6XW11VYy8ZSgbcsD9Y", "realy.example.com/repo2"},
+		{"root", "l_T9Of4ru-PLGUxxvw3SfZH0e6XW11VYy8ZSgbcsD9Y", "realy.example.com/repo2"},
-		{"mention", "p:JMkZVnu9QFplR4F_KrWX-3chQsklXZq_5I6eYcXfz1Q", "realy.example.com/repo3"},
+		{"mention", "JMkZVnu9QFplR4F_KrWX-3chQsklXZq_5I6eYcXfz1Q", "realy.example.com/repo3"},
 	}
 	var err error
 	var tgs []*tag.T
@@ -40,6 +40,7 @@ func TestT_Marshal_Unmarshal(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !bytes.Equal(m1, m2) {
+		log.I.S(m1, m2)
 		t.Fatalf("not equal:\n%s\n%s", m1, m2)
 	}
-}
+}

pkg/types/types.go

@@ -18,7 +18,7 @@ func (t *T) Marshal(d []byte) (r []byte, err error) {
 	if t == nil {
 		return
 	}
-	r = append(append(d, t.t...), '\n')
+	r = append(d, t.t...)
 	return
 }
 
@@ -39,11 +39,11 @@ func (t *T) Unmarshal(d []byte) (r []byte, err error) {
 			// write read data up to the newline and return the remainder after
 			// the newline.
 			t.t = r[:i]
-			r = r[i+1:]
+			r = r[i:]
 			return
 		}
 	}
 	// a T must end with a newline or an io.EOF is returned.
 	err = io.EOF
 	return
-}
+}

pkg/types/types_test.go

@@ -2,6 +2,8 @@ package types
 
 import (
 	"testing"
+
+	"protocol.realy.lol/pkg/separator"
 )
 
 func TestT_Marshal_Unmarshal(t *testing.T) {
@@ -11,6 +13,8 @@ func TestT_Marshal_Unmarshal(t *testing.T) {
 	if res, err = typ.Marshal(nil); chk.E(err) {
 		t.Fatal(err)
 	}
+	res = separator.Add(res)
+	log.I.S(res)
 	t2 := new(T)
 	var rem []byte
 	if rem, err = t2.Unmarshal(res); chk.E(err) {
@@ -22,4 +26,4 @@ func TestT_Marshal_Unmarshal(t *testing.T) {
 	if !typ.Equal(t2) {
 		t.Fatal("types.T did not encode/decode faithfully")
 	}
-}
+}

pkg/url/url.go

@@ -27,7 +27,7 @@ func (u *U) Equal(u2 *U) bool { return bytes.Equal(u.uu, u2.uu) }
 // Marshal a URL, use New to ensure it is valid beforehand. Appends a terminal
 // newline.
 func (u *U) Marshal(dst []byte) (result []byte, err error) {
-	result = append(append(dst, u.uu...), '\n')
+	result = append(dst, u.uu...)
 	return
 }
 
@@ -46,4 +46,4 @@ func (u *U) Unmarshal(data []byte) (rem []byte, err error) {
 		return
 	}
 	return
-}
+}

pkg/url/url_test.go

@@ -2,6 +2,8 @@ package url
 
 import (
 	"testing"
+
+	"protocol.realy.lol/pkg/separator"
 )
 
 func TestU_Marshal_Unmarshal(t *testing.T) {
@@ -15,6 +17,7 @@ func TestU_Marshal_Unmarshal(t *testing.T) {
 	if m1, err = u1.Marshal(nil); chk.E(err) {
 		t.Fatal(err)
 	}
+	m1 = separator.Add(m1)
 	u2 := new(U)
 	var rem []byte
 	if rem, err = u2.Unmarshal(m1); chk.E(err) {
@@ -26,4 +29,4 @@ func TestU_Marshal_Unmarshal(t *testing.T) {
 	if !u2.Equal(u1) {
 		t.Fatalf("u1 should be equal to u2: '%s' != '%s'", u1, u2)
 	}
-}
+}

readme.adoc

@@ -1,5 +1,11 @@
 = REALY Protocol
 :toc:
+:important-caption: 🔥
+:note-caption: 🗩
+:tip-caption: 💡
+:caution-caption: ⚠
+:table-caption: 🔍
+:example-caption: 🥚
 
 image:https://img.shields.io/badge/godoc-documentation-blue.svg[Documentation,link=https://pkg.go.dev/protocol.realy.lol]
 image:https://img.shields.io/badge/matrix-chat-green.svg[matrix chat,link=https://matrix.to/#/#realy-general:matrix.org]
@@ -229,10 +235,10 @@ As per implementation, each capability should be part of a registered list of me
 | `<type name>\n` | can be anything, hierarchic names like `note/html` `note/md` are possible, or `type.subtype` or whatever
 | `<pubkey>\n` | encoded in URL-base64 with the padding single `=` elided
 | `<unix second precision timestamp in decimal ascii>\n` |
-| `tags:\n`| Tags are a zero or more length list of lines delimited by this header and a new line after the content
+| `tags:<number>\n`| Tags has the number of tags present, and then one linebreak for each tag
 | `key:value;extra;...\n` | zero or more line separated, fields cannot contain a semicolon, end with newline instead of semicolon, key lowercase alphanumeric, first alpha, no whitespace or symbols, only key and following `:` are mandatory
 | `\n` | tags end with a double linebreak
-| `content:\n` | literally this word on one line *directly* after the newline of the previous
+| `content:<length>\n` | literally this word on one line *directly* after the newline of the previous
 | `<content>\n` | any number of further line breaks, last line is signature, everything before signature line is part of the canonical hash
 2+^| The canonical form is the above, creating the message hash that is generated with Blake 2b
 | `<ed25519 signature encoded in URL-base64>\n` | this field would have two padding chars `==`, these should be elided before generating the encoding.
@@ -424,4 +430,4 @@ The response message is like as the `filter`, the actual fetching of events is a
 |`response:fulltext\n`| each event is marked with his header, so `\nevent:` serves as a section marker
 |`<event id>\n`|  event id that matches the search terms
 |`...` | any number of events further, sorted by relevance.
-|====
+|====