completed tags codec

doc/events_queries.adoc

@@ -32,7 +32,7 @@ Event ID hashes will be encoded in URL-base64 where used in tags or mentioned in
 
 Indexing tags should be done with a truncated Blake2b hash cut at 8 bytes in the event store.
 
-Submitting an event to be stored is the same as a result sent from an Event Id query except with the type of operation inteded: `store\n` to store an event, `replace:<Event Id>\n` to replace an existing event and `relay\n` to not store but send to subscribers with open matching filters.
+Submitting an event to be stored is the same as a result sent from an Event Id query except with the type of operation inteded: `store\n` to store an event, `replace:<Event Id>\n` to replace an existing event and `relay\n` to not store but send to subscribers with open matching filters. Replace will not be accepted if the message type and pubkey are different to the original that is specified.
 
 An event is then acknowledged to be stored or rejected with a message `ok:<true/false>;<Event Id>;<reason type>:human readable part` where the reason type is one of a set of common types to indicate the reason for the false
 

pkg/content/content.go

@@ -0,0 +1,54 @@
+package content
+
+import (
+	"bytes"
+)
+
+// C is raw content bytes of a message. This can contain anything but when it is
+// unmarshalled it is assumed that the last line (content between the second
+// last and last line break) is not part of the content, as this is where the
+// signature is placed.
+//
+// The only guaranteed property of an encoded content.C is that it has two
+// newline characters, one at the very end, and a second one before it that
+// demarcates the end of the actual content. It can be entirely binary and mess
+// up a terminal to render the unsanitized possible control characters.
+type C struct{ Content []byte }
+
+// Marshal just writes the provided data with a `content:\n` prefix and adds a
+// terminal newline.
+func (c *C) Marshal(dst []byte) (result []byte, err error) {
+	result = append(append(append(dst, "content:\n"...), c.Content...), '\n')
+	return
+}
+
+var Prefix = "content:\n"
+
+// Unmarshal expects the `content:\n` prefix and stops at the second last
+// newline. The data between the second last and last newline in the data is
+// assumed to be a signature but it could be anything in another use case.
+func (c *C) Unmarshal(data []byte) (rem []byte, err error) {
+	if !bytes.HasPrefix(data, []byte("content:\n")) {
+		err = errorf.E("content prefix `content:\\n' not found: '%s'", data[:len(Prefix)+1])
+		return
+	}
+	// trim off the prefix.
+	data = data[len(Prefix):]
+	// check that there is a last newline.
+	if data[len(data)-1] != '\n' {
+		err = errorf.E("input data does not end with newline")
+		return
+	}
+	// we start at the second last, previous to the terminal newline byte.
+	lastPos := len(data) - 2
+	for ; lastPos >= len(Prefix); lastPos-- {
+		// the content ends at the byte before the second last newline byte.
+		if data[lastPos] == '\n' {
+			break
+		}
+	}
+	c.Content = data[:lastPos]
+	// return the remainder after the content-terminal newline byte.
+	rem = data[lastPos+1:]
+	return
+}

pkg/content/content_test.go

@@ -0,0 +1,39 @@
+package content
+
+import (
+	"bytes"
+	"crypto/rand"
+	mrand "math/rand"
+	"testing"
+)
+
+func TestC_Marshal_Unmarshal(t *testing.T) {
+	c := make([]byte, mrand.Intn(100)+25)
+	_, err := rand.Read(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	log.I.S(c)
+	c1 := new(C)
+	c1.Content = c
+	var res []byte
+	if res, err = c1.Marshal(nil); chk.E(err) {
+		t.Fatal(err)
+	}
+	// append a fake zero length signature
+	res = append(res, '\n')
+	log.I.S(res)
+	c2 := new(C)
+	var rem []byte
+	if rem, err = c2.Unmarshal(res); chk.E(err) {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(c1.Content, c2.Content) {
+		log.I.S(c1, c2)
+		t.Fatal("content not equal")
+	}
+	if !bytes.Equal(rem, []byte{'\n'}) {
+		log.I.S(rem)
+		t.Fatalf("remainder not found")
+	}
+}

pkg/content/log.go

@@ -0,0 +1,9 @@
+package content
+
+import (
+	"protocol.realy.lol/pkg/lol"
+)
+
+var (
+	log, chk, errorf = lol.Main.Log, lol.Main.Check, lol.Main.Errorf
+)

pkg/event/event.go

@@ -1,16 +1,17 @@
 package event
 
 import (
+	"protocol.realy.lol/pkg/content"
 	"protocol.realy.lol/pkg/event/types"
 	"protocol.realy.lol/pkg/pubkey"
 	"protocol.realy.lol/pkg/signature"
 )
 
 type Event struct {
-	Type      types.T
-	Pubkey    pubkey.P
+	Type      *types.T
+	Pubkey    *pubkey.P
 	Timestamp int64
 	Tags      [][]byte
-	Content   []byte
-	Signature signature.S
+	Content   *content.C
+	Signature *signature.S
 }

pkg/id/id.go

@@ -0,0 +1,74 @@
+package id
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"encoding/base64"
+	"io"
+)
+
+const Len = 43
+
+type P struct{ b []byte }
+
+func New(id []byte) (p *P, err error) {
+	if len(id) != ed25519.PublicKeySize {
+		err = errorf.E("invalid public key size: %d; require %d",
+			len(id), ed25519.PublicKeySize)
+		return
+	}
+	p = &P{id}
+	return
+}
+
+func (p *P) Marshal(dst []byte) (result []byte, err error) {
+	result = dst
+	if p == nil || p.b == nil || len(p.b) == 0 {
+		err = errorf.E("nil/zero length pubkey")
+		return
+	}
+	if len(p.b) != ed25519.PublicKeySize {
+		err = errorf.E("invalid public key length %d; require %d '%0x'",
+			len(p.b), ed25519.PublicKeySize, p.b)
+		return
+	}
+	buf := bytes.NewBuffer(result)
+	w := base64.NewEncoder(base64.RawURLEncoding, buf)
+	if _, err = w.Write(p.b); chk.E(err) {
+		return
+	}
+	if err = w.Close(); chk.E(err) {
+		return
+	}
+	result = append(buf.Bytes(), '\n')
+	return
+}
+
+func (p *P) Unmarshal(data []byte) (rem []byte, err error) {
+	rem = data
+	if p == nil {
+		err = errorf.E("can't unmarshal into nil types.T")
+		return
+	}
+	if len(rem) < 2 {
+		err = errorf.E("can't unmarshal nothing")
+		return
+	}
+	for i := range rem {
+		if rem[i] == '\n' {
+			if i != Len {
+				err = errorf.E("invalid encoded pubkey length %d; require %d '%0x'",
+					i, Len, rem[:i])
+				return
+			}
+			p.b = make([]byte, ed25519.PublicKeySize)
+			if _, err = base64.RawURLEncoding.Decode(p.b, rem[:i]); chk.E(err) {
+				return
+			}
+			rem = rem[i+1:]
+			return
+		}
+	}
+	err = io.EOF
+	return
+}

pkg/id/id_test.go

@@ -0,0 +1,40 @@
+package id
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"crypto/rand"
+	"testing"
+)
+
+func TestP_Marshal_Unmarshal(t *testing.T) {
+	var err error
+	for range 10 {
+		pk := make([]byte, ed25519.PublicKeySize)
+		if _, err = rand.Read(pk); chk.E(err) {
+			t.Fatal(err)
+		}
+		log.I.S(pk)
+		var p *P
+		if p, err = New(pk); chk.E(err) {
+			t.Fatal(err)
+		}
+		var o []byte
+		if o, err = p.Marshal(nil); chk.E(err) {
+			t.Fatal(err)
+		}
+		log.I.F("%d %s", len(o), o)
+		p2 := &P{}
+		var rem []byte
+		if rem, err = p2.Unmarshal(o); chk.E(err) {
+			t.Fatal(err)
+		}
+		if len(rem) > 0 {
+			log.I.F("%d %s", len(rem), rem)
+		}
+		log.I.S(p2.b)
+		if !bytes.Equal(pk, p2.b) {
+			t.Fatal("public key did not encode/decode faithfully")
+		}
+	}
+}

pkg/id/log.go

@@ -0,0 +1,9 @@
+package id
+
+import (
+	"protocol.realy.lol/pkg/lol"
+)
+
+var (
+	log, chk, errorf = lol.Main.Log, lol.Main.Check, lol.Main.Errorf
+)

pkg/tag/log.go

@@ -0,0 +1,9 @@
+package tag
+
+import (
+	"protocol.realy.lol/pkg/lol"
+)
+
+var (
+	log, chk, errorf = lol.Main.Log, lol.Main.Check, lol.Main.Errorf
+)

pkg/tag/tag.go

@@ -0,0 +1,135 @@
+// Package tag defines a format for event tags that follows the following rules:
+//
+// First field is the key, this is to be hashed using Blake2b and truncated to 8 bytes for indexing. These keys should
+// not be long, and thus will not have any collisions as a truncated hash. The terminal byte of a key is the colon `:`
+//
+// Subsequent fields are separated by semicolon ';' and they can contain any data except a semicolon or newline.
+//
+// The tag is terminated by a newline.
+package tag
+
+import (
+	"bytes"
+)
+
+type fields [][]byte
+
+type T struct{ fields }
+
+func New[V ~[]byte | ~string](v ...V) (t *T, err error) {
+	t = new(T)
+	var k []byte
+	if k, err = ValidateKey([]byte(v[0])); err != nil {
+		err = errorf.E("")
+		return
+	}
+	v = v[1:]
+	t.fields = append(t.fields, k)
+	for i, val := range v {
+		var b []byte
+		if b, err = ValidateField(val, i); chk.E(err) {
+			return
+		}
+		t.fields = append(t.fields, b)
+	}
+	return
+}
+
+// ValidateKey checks that the key is valid. Keys must be the same most language symbols:
+//
+// - first character is alphabetic [a-zA-Z]
+// - subsequent characters can be alphanumeric and underscore [a-zA-Z0-9_]
+//
+// If the key is not valid this function returns a nil value.
+func ValidateKey[V ~[]byte | ~string](key V) (k []byte, err error) {
+	if len(key) < 1 {
+		return
+	}
+	kb := []byte(key)
+	switch {
+	case kb[0] < 'a' && k[0] > 'z' || kb[0] < 'A' && kb[0] > 'Z':
+		for i, b := range kb[1:] {
+			switch {
+			case (b > 'a' && b < 'z') || b > 'A' && b < 'Z' || b == '_' || b > '0' && b < '9':
+			default:
+				err = errorf.E("invalid character in tag key at index %d '%c': \"%s\"", i, b, kb)
+				return
+			}
+		}
+	}
+	// if we got to here, the whole string is compliant
+	k = kb
+	return
+}
+
+func ValidateField[V ~[]byte | ~string](f V, i int) (k []byte, err error) {
+	b := []byte(f)
+	if bytes.Contains(b, []byte(";")) {
+		err = errorf.E("key %d cannot contain ';': '%s'", i, b)
+		return
+	}
+	if bytes.Contains(b, []byte("\n")) {
+		err = errorf.E("key %d cannot contain '\\n': '%s'", i, b)
+		return
+	}
+	// if we got to here, the whole string is compliant
+	k = b
+	return
+}
+
+func (t *T) Marshal(dst []byte) (result []byte, err error) {
+	result = dst
+	if len(t.fields) == 0 {
+		return
+	}
+	for i, field := range t.fields {
+		result = append(result, field...)
+		if i == 0 {
+			result = append(result, ':')
+		} else if i == len(t.fields)-1 {
+			result = append(result, '\n')
+		} else {
+			result = append(result, ';')
+		}
+	}
+	return
+}
+
+func (t *T) Unmarshal(data []byte) (rem []byte, err error) {
+	var i int
+	var v byte
+	var dat []byte
+	// first find the end
+	for i, v = range data {
+		if v == '\n' {
+			dat, rem = data[:i], data[i+1:]
+			break
+		}
+	}
+	if len(dat) == 0 {
+		err = errorf.E("invalid empty tag")
+		return
+	}
+	for i, v = range dat {
+		if v == ':' {
+			f := dat[:i]
+			dat = dat[i+1:]
+			t.fields = append(t.fields, f)
+			break
+		}
+	}
+	for len(dat) > 0 {
+		for i, v = range dat {
+			if v == ';' {
+				t.fields = append(t.fields, dat[:i])
+				dat = dat[i+1:]
+				break
+			}
+			if i == len(dat)-1 {
+				t.fields = append(t.fields, dat)
+				return
+			}
+		}
+	}
+	return
+}

pkg/tag/tag_test.go

@@ -0,0 +1,29 @@
+package tag
+
+import (
+	"testing"
+)
+
+func TestT_Marshal_Unmarshal(t *testing.T) {
+	var err error
+	var t1 *T
+	if t1, err = New("reply", "e:l_T9Of4ru-PLGUxxvw3SfZH0e6XW11VYy8ZSgbcsD9Y",
+		"realy.example.com/repo"); chk.E(err) {
+		t.Fatal(err)
+	}
+	var tb []byte
+	if tb, err = t1.Marshal(nil); chk.E(err) {
+		t.Fatal(err)
+	}
+	log.I.S(tb)
+	t2 := new(T)
+	var rem []byte
+	if rem, err = t2.Unmarshal(tb); chk.E(err) {
+		t.Fatal(err)
+	}
+	if len(rem) > 0 {
+		log.I.F("%s", rem)
+		t.Fatal("remainder after tag should have been nothing")
+	}
+	log.I.S(t2)
+}

pkg/tags/log.go

@@ -0,0 +1,9 @@
+package tags
+
+import (
+	"protocol.realy.lol/pkg/lol"
+)
+
+var (
+	log, chk, errorf = lol.Main.Log, lol.Main.Check, lol.Main.Errorf
+)

pkg/tags/tags.go

@@ -0,0 +1,56 @@
+package tags
+
+import (
+	"bytes"
+	"fmt"
+
+	"protocol.realy.lol/pkg/tag"
+)
+
+const Sentinel = "tags:\n"
+
+var SentinelBytes = []byte(Sentinel)
+
+type tags []*tag.T
+
+type T struct{ tags }
+
+func New(v ...*tag.T) *T { return &T{tags: v} }
+
+func (t *T) Marshal(dst []byte) (result []byte, err error) {
+	result = dst
+	result = append(result, Sentinel...)
+	for _, tt := range t.tags {
+		if result, err = tt.Marshal(result); chk.E(err) {
+			return
+		}
+	}
+	result = append(result, '\n')
+	return
+}
+
+func (t *T) Unmarshal(data []byte) (rem []byte, err error) {
+	if len(data) < len(Sentinel) {
+		err = fmt.Errorf("bytes too short to contain tags")
+		return
+	}
+	var dat []byte
+	if bytes.Equal(data[:len(Sentinel)], SentinelBytes) {
+		dat = data[len(Sentinel):]
+	}
+	if len(dat) < 1 {
+		return
+	}
+	for len(dat) > 0 {
+		if len(dat) == 1 && dat[0] == '\n' {
+			break
+		}
+		// log.I.S(dat)
+		tt := new(tag.T)
+		if dat, err = tt.Unmarshal(dat); chk.E(err) {
+			return
+		}
+		t.tags = append(t.tags, tt)
+	}
+	return
+}

pkg/tags/tags_test.go

@@ -0,0 +1,45 @@
+package tags
+
+import (
+	"bytes"
+	"testing"
+
+	"protocol.realy.lol/pkg/tag"
+)
+
+func TestT_Marshal_Unmarshal(t *testing.T) {
+	var tegs = [][]string{
+		{"reply", "e:l_T9Of4ru-PLGUxxvw3SfZH0e6XW11VYy8ZSgbcsD9Y", "realy.example.com/repo1"},
+		{"root", "e:l_T9Of4ru-PLGUxxvw3SfZH0e6XW11VYy8ZSgbcsD9Y", "realy.example.com/repo2"},
+		{"mention", "p:JMkZVnu9QFplR4F_KrWX-3chQsklXZq_5I6eYcXfz1Q", "realy.example.com/repo3"},
+	}
+	var err error
+	var tgs []*tag.T
+	for _, teg := range tegs {
+		var tg *tag.T
+		if tg, err = tag.New(teg...); chk.E(err) {
+			t.Fatal(err)
+		}
+		tgs = append(tgs, tg)
+	}
+	t1 := New(tgs...)
+	var m1 []byte
+	if m1, err = t1.Marshal(nil); chk.E(err) {
+		t.Fatal(err)
+	}
+	t2 := new(T)
+	var rem []byte
+	if rem, err = t2.Unmarshal(m1); chk.E(err) {
+		t.Fatal(err)
+	}
+	if len(rem) > 0 {
+		t.Fatalf("%s", rem)
+	}
+	var m2 []byte
+	if m2, err = t2.Marshal(nil); chk.E(err) {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(m1, m2) {
+		t.Fatalf("not equal:\n%s\n%s", m1, m2)
+	}
+}