all types working correctly

2025-05-21 22:42:51 -01:06
parent 6a9b57af01
commit c8696d7417
22 changed files with 1358 additions and 135 deletions
--- a/content/content.go
+++ b/content/content.go
@@ -6,6 +6,7 @@ import (

 	"github.com/mleku/transit/chk"
 	"github.com/mleku/transit/errorf"
+	"github.com/mleku/transit/text"
 )

 var BinaryPrefix = []byte("base64:")
@@ -29,12 +30,11 @@ func (c *C) MarshalJSON() (b []byte, err error) {
 		b = make([]byte, base64.URLEncoding.EncodedLen(len(c.Val))+2+len(BinaryPrefix))
 		copy(b[1:], BinaryPrefix)
 		base64.URLEncoding.Encode(b[1+len(BinaryPrefix):len(b)-1], c.Val)
+		b[0] = '"'
+		b[len(b)-1] = '"'
 	} else {
-		b = make([]byte, len(c.Val)+2)
-		copy(b[1:], c.Val)
+		b = append(append([]byte{'"'}, text.Escape(nil, c.Val)...), '"')
 	}
-	b[0] = '"'
-	b[len(b)-1] = '"'
 	return
 }

@@ -54,6 +54,7 @@ func (c *C) UnmarshalJSON(b []byte) (err error) {
 	} else {
 		b = b[:len(b)-1]
 		c.Val = b
+		c.Val = text.Unescape(c.Val)
 		c.Binary = false
 	}
 	return
--- a/content/content_test.go
+++ b/content/content_test.go
@@ -12,15 +12,17 @@ import (
 )

 func TestNew(t *testing.T) {
-	content := "this is some plain text"
-	log.I.F("%s", content)
-	c := New(content, false)
 	var err error
 	var b []byte
+	content := `this is some plain text
+	
+	with a newline`
+	c := New(content, false)
+	log.I.F("%s", c.Val)
 	if b, err = json.Marshal(&c); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.F("%s", b)
+	// log.I.F("%s", b)
 	c2 := &C{}
 	if err = json.Unmarshal(b, c2); chk.E(err) {
 		t.Fatal(err)
@@ -34,16 +36,15 @@ func TestNew(t *testing.T) {
 		t.Fatal(err)
 	}
 	bin1 := New(bin, true)
+	log.I.S(bin1)
 	if b, err = json.Marshal(bin1); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.S(bin1)
-	log.I.S(b)
 	bin2 := &C{}
 	if err = json.Unmarshal(b, bin2); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.S(bin2)
+	log.I.S(bin1.Val, bin2.Val)
 	if !bytes.Equal(bin1.Val, bin2.Val) {
 		t.Fatal("failed to encode/decode")
 	}
--- a/event/event.go
+++ b/event/event.go
@@ -1,27 +0,0 @@
-package event
-
-import (
-	"github.com/mleku/transit/content"
-	"github.com/mleku/transit/id"
-	"github.com/mleku/transit/kind"
-	"github.com/mleku/transit/mime"
-	"github.com/mleku/transit/tag"
-)
-
-// E is an event on the transit protocol.
-type E struct {
-	// Id is the hash of the canonical format of the event.E.
-	Id *id.I `json:"id" doc:"hash of canonical form of event in unpadded base64 URL encoding"`
-	// Mimetype is the type of data contained in the Content field of an event.E.
-	Mimetype *mime.Type `json:"mimetype" doc:"standard mimetype descriptor for the format of the content field"`
-	// Kind is a short name of the application/intent of the event.
-	Kind *kind.K `json:"kind" doc:"short name of the application/intent of the event"`
-	// Content is the payload of the event.E.
-	Content *content.C
-	// Tags are a set of descriptors formatted as a key, value and optional extra fields.
-	Tags tag.S `json:"tags" doc:"collection of tags with a short key name and value"`
-}
-
-// C is the canonical format for an event.E that is hashed to generate the id.I.
-type C struct {
-}
--- a/go.mod
+++ b/go.mod
@@ -5,14 +5,16 @@ go 1.24.3
 require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/fatih/color v1.18.0
+	github.com/minio/sha256-simd v1.0.1
+	github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b
 	github.com/zeebo/blake3 v0.2.4
-	golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6
 	lukechampine.com/frand v1.5.1
 )

 require (
-	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/templexxx/cpu v0.0.1 // indirect
 	golang.org/x/sys v0.25.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -2,21 +2,26 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
-github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
-github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
+github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
+github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
+github.com/templexxx/cpu v0.0.1 h1:hY4WdLOgKdc8y13EYklu9OUTXik80BkxHoWvTO6MQQY=
+github.com/templexxx/cpu v0.0.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk=
+github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b h1:XeDLE6c9mzHpdv3Wb1+pWBaWv/BlHK0ZYIu/KaL6eHg=
+github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b/go.mod h1:7rwmCH0wC2fQvNEvPZ3sKXukhyCTyiaZ5VTZMQYpZKQ=
 github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
 github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
 github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
 github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
 github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
-golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI=
-golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
--- a/hex/aliases.go
+++ b/hex/aliases.go
@@ -0,0 +1,51 @@
+// Package hex is a set of aliases and helpers for using the templexxx SIMD hex
+// encoder.
+package hex
+
+import (
+	"encoding/hex"
+
+	"github.com/templexxx/xhex"
+
+	"github.com/mleku/transit/chk"
+	"github.com/mleku/transit/errorf"
+)
+
+var Enc = hex.EncodeToString
+var EncBytes = hex.Encode
+var Dec = hex.DecodeString
+var DecBytes = hex.Decode
+
+// var EncAppend = hex.AppendEncode
+// var DecAppend = hex.AppendDecode
+
+var DecLen = hex.DecodedLen
+
+type InvalidByteError = hex.InvalidByteError
+
+// EncAppend uses xhex to encode a sice of bytes and appends it to a provided destination slice.
+func EncAppend(dst, src []byte) (b []byte) {
+	l := len(dst)
+	dst = append(dst, make([]byte, len(src)*2)...)
+	xhex.Encode(dst[l:], src)
+	return dst
+}
+
+// DecAppend decodes a provided encoded hex encoded string and appends the decoded output to a
+// provided input slice.
+func DecAppend(dst, src []byte) (b []byte, err error) {
+	if src == nil || len(src) < 2 {
+		err = errorf.E("nothing to decode")
+		return
+	}
+	if dst == nil {
+		dst = []byte{}
+	}
+	l := len(dst)
+	b = dst
+	b = append(b, make([]byte, len(src)/2)...)
+	if err = xhex.Decode(b[l:], src); chk.T(err) {
+		return
+	}
+	return
+}
--- a/kind/kind_test.go
+++ b/kind/kind_test.go
@@ -10,7 +10,7 @@ import (
 )

 func TestNew(t *testing.T) {
-	kind := "reply"
+	kind := "forum/reply"
 	log.I.F("%s", kind)
 	k := New(kind)
 	var err error
--- a/lol/log.go
+++ b/lol/log.go
@@ -183,7 +183,7 @@ var msgCol = color.New(color.FgBlue).Sprint
 func GetPrinter(l int32, writer io.Writer, skip int) LevelPrinter {
 	return LevelPrinter{
 		Ln: func(a ...interface{}) {
-			if Level.Load() < l {
+			if Level.Load() > l {
 				return
 			}
 			fmt.Fprintf(writer,
@@ -195,7 +195,7 @@ func GetPrinter(l int32, writer io.Writer, skip int) LevelPrinter {
 			)
 		},
 		F: func(format string, a ...interface{}) {
-			if Level.Load() < l {
+			if Level.Load() > l {
 				return
 			}
 			fmt.Fprintf(writer,
@@ -207,7 +207,7 @@ func GetPrinter(l int32, writer io.Writer, skip int) LevelPrinter {
 			)
 		},
 		S: func(a ...interface{}) {
-			if Level.Load() < l {
+			if Level.Load() > l {
 				return
 			}
 			fmt.Fprintf(writer,
@@ -219,7 +219,7 @@ func GetPrinter(l int32, writer io.Writer, skip int) LevelPrinter {
 			)
 		},
 		C: func(closure func() string) {
-			if Level.Load() < l {
+			if Level.Load() > l {
 				return
 			}
 			fmt.Fprintf(writer,
@@ -231,7 +231,7 @@ func GetPrinter(l int32, writer io.Writer, skip int) LevelPrinter {
 			)
 		},
 		Chk: func(e error) bool {
-			if Level.Load() < l {
+			if Level.Load() > l {
 				return e != nil
 			}
 			if e != nil {
@@ -247,15 +247,15 @@ func GetPrinter(l int32, writer io.Writer, skip int) LevelPrinter {
 			return false
 		},
 		Err: func(format string, a ...interface{}) error {
-			// if Level.Load() < l {
-			fmt.Fprintf(writer,
-				"%s%s %s %s\n",
-				msgCol(TimeStamper()),
-				LevelSpecs[l].Colorizer(LevelSpecs[l].Name, " "),
-				fmt.Sprintf(format, a...),
-				msgCol(GetLoc(skip)),
-			)
-			// }
+			if Level.Load() > l {
+				fmt.Fprintf(writer,
+					"%s%s %s %s\n",
+					msgCol(TimeStamper()),
+					LevelSpecs[l].Colorizer(LevelSpecs[l].Name, " "),
+					fmt.Sprintf(format, a...),
+					msgCol(GetLoc(skip)),
+				)
+			}
 			return fmt.Errorf(format, a...)
 		},
 	}
--- a/mime/mime_test.go
+++ b/mime/mime_test.go
--- a/signature/signature.go
+++ b/signature/signature.go
@@ -0,0 +1,50 @@
+package signature
+
+import (
+	"crypto/ed25519"
+	"encoding/base64"
+
+	"github.com/mleku/transit/chk"
+	"github.com/mleku/transit/errorf"
+)
+
+const Len = ed25519.SignatureSize
+
+var EncodedLen = base64.RawURLEncoding.EncodedLen(Len)
+
+// S is a blake3 hash encoded in base64 URL format.
+type S struct {
+	Bytes []byte
+}
+
+// New creates a new id.I based on a provided message bytes.
+func New[V []byte | string](sig V) (i *S) {
+	i = &S{Bytes: []byte(sig)}
+	return
+}
+
+func (i *S) MarshalJSON() (b []byte, err error) {
+	if len(i.Bytes) != Len {
+		err = errorf.E("id must be %d bytes long, got %d", Len, len(i.Bytes))
+		return
+	}
+	b = make([]byte, EncodedLen+2)
+	b[0] = '"'
+	b[len(b)-1] = '"'
+	base64.RawURLEncoding.Encode(b[1:], i.Bytes)
+	return
+}
+
+func (i *S) UnmarshalJSON(b []byte) (err error) {
+	if len(b) != EncodedLen+2 {
+		err = errorf.E("encoded signature must be %d bytes long, got %d", EncodedLen+2, len(b))
+		return
+	}
+	i.Bytes = make([]byte, Len)
+	var n int
+	if n, err = base64.RawURLEncoding.Decode(i.Bytes, b[1:EncodedLen+1]); chk.E(err) {
+		err = errorf.E("error: signature decoding failed at %d: %s", n, err.Error())
+		return
+	}
+	return
+}
--- a/signature/signature_test.go
+++ b/signature/signature_test.go
@@ -0,0 +1,31 @@
+package signature
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"lukechampine.com/frand"
+
+	"github.com/mleku/transit/chk"
+	"github.com/mleku/transit/log"
+)
+
+func TestNew(t *testing.T) {
+	msg := frand.Bytes(Len)
+	i := New(msg)
+	var err error
+	var b []byte
+	if b, err = json.Marshal(&i); chk.E(err) {
+		t.Fatal(err)
+	}
+	log.I.F("encode %0x %d %s", i.Bytes, len(b), b)
+	i2 := &S{}
+	if err = json.Unmarshal(b, i2); chk.E(err) {
+		t.Fatal(err)
+	}
+	log.I.F("decode %0x", i2.Bytes)
+	if !bytes.Equal(i.Bytes, i2.Bytes) {
+		t.Fatal("failed to encode/decode")
+	}
+}
--- a/tag/tag.go
+++ b/tag/tag.go
@@ -13,32 +13,60 @@ import (

 type T [2]any

-// Types is a list of types of tags that are valid. Note that this is all of the types that are
+type U struct{ T }
+
+// MarshalTypes is a list of types of tags that are valid. Note that this is all of the types that are
 // valid. Additional tag keys may be added in the future but should never be taken away.
-var Types = map[string]reflect.Type{
+var MarshalTypes = map[string]reflect.Type{
 	"root":    reflect.TypeOf(&id.I{}),
 	"parent":  reflect.TypeOf(&id.I{}),
 	"event":   reflect.TypeOf(&id.I{}),
 	"user":    reflect.TypeOf(&pubkey.P{}),
 	"hashtag": reflect.TypeOf(""),
-	"url":     reflect.TypeOf(&url.URL{}),
+	"url":     reflect.TypeOf(""),
 }

-// New is used to return an empty type for unmarshalling.
-func New(t string) any { return Types[t] }
+// UnmarshalTypes is a list of types of tags that are valid. Note that this is all of the types that are
+// valid. Additional tag keys may be added in the future but should never be taken away.
+var UnmarshalTypes = map[string]func() any{
+	"root":    func() any { return id.I{} },
+	"parent":  func() any { return id.I{} },
+	"event":   func() any { return id.I{} },
+	"user":    func() any { return pubkey.P{} },
+	"hashtag": func() any { return "" },
+	"url":     func() any { return "" },
+}

-func (t T) MarshalJSON() (b []byte, err error) {
+func New(key string, val any) (t *U) {
+	return &U{T{key, val}}
+}
+
+// GetId returns the correctly typed value or the ok value is false.
+func (t *U) GetId() (i *id.I, ok bool) { i, ok = t.T[1].(*id.I); return }
+
+// GetPubkey returns the correctly typed value or the ok value is false.
+func (t *U) GetPubkey() (p *pubkey.P, ok bool) {
+	p, ok = t.T[1].(*pubkey.P)
+	return
+}
+
+// GetString returns the correctly typed value or the ok value is false.
+func (t *U) GetString() (s string, ok bool) {
+	s, ok = t.T[1].(string)
+	return
+}
+
+func (t *U) MarshalJSON() (b []byte, err error) {
 	// first value must be a string
-	k, ok := t[0].(string)
-	chk.E(err)
+	k, ok := t.T[0].(string)
 	if !ok {
 		err = errorf.E("key must be a string, got '%v'", k)
 		return
 	}
-	// second value must be one of Types.
+	// second value must be one of MarshalTypes.
 	var found bool
-	for _, v := range Types {
-		if v == reflect.TypeOf(t[1]) {
+	for _, v := range MarshalTypes {
+		if v == reflect.TypeOf(t.T[1]) {
 			found = true
 			break
 		}
@@ -46,38 +74,40 @@ func (t T) MarshalJSON() (b []byte, err error) {
 	// url is a special case because it needs to be rendered using a stringer as it lacks a
 	// json.MarshalJSON implementation.
 	if k == "url" {
-		var u *url.URL
-		if u, ok = t[1].(*url.URL); !ok {
-			err = errorf.E("url key does not have a value of type *url.URL: %v",
-				reflect.TypeOf(t[1]))
+		var ur string
+		if ur, ok = t.T[1].(string); !ok {
+			err = errorf.E("url key does not have a value of type string: %v",
+				reflect.TypeOf(t.T[1]))
 			return
 		}
-		found = true
-		t[1] = u.String()
+		if _, err = url.Parse(ur); chk.E(err) {
+			return
+		}
+		t.T[1] = ur
 	}
 	if k == "hashtag" {
 		var ht string
-		if ht, ok = t[1].(string); !ok {
+		if ht, ok = t.T[1].(string); !ok {
 			err = errorf.E("hashtag key does not have a value of type []byte: %v",
-				reflect.TypeOf(t[1]))
+				reflect.TypeOf(t.T[1]))
 			return
 		}
-		t[1] = string(ht)
+		t.T[1] = ht
 	}
 	if !found {
 		err = errorf.E("tag value is unknown type key: %s value: %v",
-			t[0], reflect.TypeOf(t[1]))
+			t.T[0], reflect.TypeOf(t.T[1]))
 		return
 	}
 	// marshal array into tag
-	tt := [2]any(t)
-	if b, err = json.Marshal(tt); chk.E(err) {
+	tt := t.T
+	if b, err = json.Marshal(&tt); chk.E(err) {
 		return
 	}
 	return
 }

-func (t T) UnmarshalJSON(b []byte) (err error) {
+func (t *U) UnmarshalJSON(b []byte) (err error) {
 	var kv []any
 	if err = json.Unmarshal(b, &kv); chk.E(err) {
 		return
@@ -85,32 +115,43 @@ func (t T) UnmarshalJSON(b []byte) (err error) {
 	// we only do the above to get the key name
 	var ok bool
 	var k string
-	if k, ok = t[0].(string); !ok {
+	if k, ok = kv[0].(string); !ok {
 		err = errorf.E("key must be a string, got '%v'", k)
 		return
 	}
-	if _, ok = Types[k]; !ok {
+	t.T[0] = k
+	var vt func() any
+	if vt, ok = UnmarshalTypes[k]; !ok {
 		err = errorf.E("unknown key type '%s'", k)
 		return
 	}
-	// the value could be any of the valid types, but all are strings listed in Types.
-	switch vt := t[1].(type) {
-	case *url.URL:
-		_ = vt
-		// if u, err = url.Parse(vt); chk.E(err) {
-		// 	return
-		// }
-		//
-		// case
-		// ty := New(k)
-		// if err = json.Unmarshal([]byte(t[1].(string)), &ty); chk.E(err) {
-		// 	return
-		// }
-		// t[1] = vt
-	default:
-		err = errorf.E("invalid type in value field: %v", reflect.TypeOf(t[1]))
+	tt := vt()
+	var vs string
+	if vs, ok = kv[1].(string); !ok {
+		err = errorf.E("value must be a string, got '%v'", k)
 		return
 	}
 	// decode in accordance with the key
+	switch ttt := tt.(type) {
+	case id.I:
+		vs = `"` + vs + `"`
+		if err = ttt.UnmarshalJSON([]byte(vs)); chk.E(err) {
+			return
+		}
+		t.T[1] = &ttt
+	case pubkey.P:
+		vs = `"` + vs + `"`
+		if err = ttt.UnmarshalJSON([]byte(vs)); chk.E(err) {
+			return
+		}
+		t.T[1] = &ttt
+	case string:
+		t.T[1] = vs
+	case *url.URL:
+		if ttt, err = url.Parse(vs); chk.E(err) {
+			return
+		}
+		t.T[1] = vs
+	}
 	return
 }
--- a/tag/tag_test.go
+++ b/tag/tag_test.go
@@ -1,64 +1,123 @@
 package tag

 import (
+	"bytes"
 	"crypto/ed25519"
 	"encoding/json"
-	"net/url"
+	"reflect"
 	"testing"

 	"lukechampine.com/frand"

 	"github.com/mleku/transit/chk"
 	"github.com/mleku/transit/id"
-	"github.com/mleku/transit/log"
 	"github.com/mleku/transit/pubkey"
 )

 func TestNew(t *testing.T) {
 	// root
-	root := T{"root", generateId()}
 	var b []byte
 	var err error
+	rootId := generateId()
+	root := New("root", rootId)
 	if b, err = json.Marshal(&root); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.F("%s", b)
-	// parent
-	parent := T{"parent", generateId()}
+	root2 := &U{}
+	if err = json.Unmarshal(b, root2); chk.E(err) {
+		t.Fatal(err)
+	}
+	if i, ok := root2.GetId(); ok {
+		if !bytes.Equal(i.Hash, rootId.Hash) {
+			t.Fatal("did not get same root id back")
+		}
+	} else {
+		t.Fatalf("got wrong type %v", reflect.TypeOf(root2.T[1]))
+	}
+	// // parent
+	parentId := generateId()
+	parent := New("parent", parentId)
 	if b, err = json.Marshal(&parent); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.F("%s", b)
+	parent2 := &U{}
+	if err = json.Unmarshal(b, parent2); chk.E(err) {
+		t.Fatal(err)
+	}
+	if i, ok := parent2.GetId(); ok {
+		if !bytes.Equal(i.Hash, parentId.Hash) {
+			t.Fatal("did not get same parent id back")
+		}
+	} else {
+		t.Fatalf("got wrong type %v", reflect.TypeOf(parent2.T[1]))
+	}
 	// event
-	eid := T{"event", generateId()}
+	eventId := generateId()
+	eid := New("event", eventId)
 	if b, err = json.Marshal(&eid); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.F("%s", b)
+	eid2 := &U{}
+	if err = json.Unmarshal(b, eid2); chk.E(err) {
+		t.Fatal(err)
+	}
+	if i, ok := eid2.GetId(); ok {
+		if !bytes.Equal(i.Hash, eventId.Hash) {
+			t.Fatal("did not get same event id back")
+		}
+	} else {
+		t.Fatalf("got wrong type %v", reflect.TypeOf(eid2.T[1]))
+	}
 	// user
-	pk := T{"user", generatePubkey()}
+	pkB := generatePubkey()
+	pk := New("user", pkB)
 	if b, err = json.Marshal(&pk); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.F("%s", b)
+	pk2 := &U{}
+	if err = json.Unmarshal(b, pk2); chk.E(err) {
+		t.Fatal(err)
+	}
+	if p, ok := pk2.GetPubkey(); ok {
+		if !bytes.Equal(p.Key, pkB.Key) {
+			t.Fatal("did not get same pubkey back")
+		}
+	} else {
+		t.Fatalf("got wrong type %v", reflect.TypeOf(pk2.T[1]))
+	}
 	// hashtag
-	hashtag := "winning"
-	ht := T{"hashtag", hashtag}
+	ht := New("hashtag", "winning")
 	if b, err = json.Marshal(&ht); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.F("%s", b)
+	hashtag2 := &U{}
+	if err = json.Unmarshal(b, hashtag2); chk.E(err) {
+		t.Fatal(err)
+	}
+	if h, ok := hashtag2.GetString(); ok {
+		if h != "winning" {
+			t.Fatal("did not get same hashtag back")
+		}
+	} else {
+		t.Fatalf("got wrong type %v", reflect.TypeOf(hashtag2.T[1]))
+	}
 	// url
-	var u *url.URL
-	if u, err = url.Parse("http://example.com"); chk.E(err) {
+	ur := "http://example.com"
+	ut := New("url", ur)
+	if b, err = json.Marshal(ut); chk.E(err) {
 		t.Fatal(err)
 	}
-	ut := T{"url", u}
-	if b, err = json.Marshal(&ut); chk.E(err) {
+	ut2 := &U{}
+	if err = json.Unmarshal(b, ut2); chk.E(err) {
 		t.Fatal(err)
 	}
-	log.I.F("%s", b)
-
+	if utt, ok := ut2.GetString(); ok {
+		if utt != ur {
+			t.Fatal("did not get same URL back")
+		}
+	} else {
+		t.Fatalf("got wrong type %v", reflect.TypeOf(ut2.T[1]))
+	}
 }

 func generateId() (i *id.I) { return id.New(frand.Bytes(32)) }
--- a/tag/tags.go
+++ b/tag/tags.go
@@ -1,5 +1,49 @@
 package tag

-type S []T
+import (
+	"encoding/json"

-func NewTags(t ...T) (s S) { return S(t) }
+	"github.com/mleku/transit/chk"
+)
+
+type Ts []*U
+type S struct{ Ts }
+
+func NewTags(t ...*U) (s *S) { return &S{Ts: t} }
+
+func (s *S) MarshalJSON() (b []byte, err error) {
+	b = append(b, '[')
+	for i, v := range s.Ts {
+		if i > 0 {
+			b = append(b, ',')
+		}
+		// b = append(b, '"')
+		var bb []byte
+		if bb, err = json.Marshal(v); chk.E(err) {
+			return
+		}
+		b = append(b, bb...)
+		// b = append(b, '"')
+	}
+	b = append(b, ']')
+	return
+}
+
+func (s *S) UnmarshalJSON(b []byte) (err error) {
+	var ss [][]string
+	if err = json.Unmarshal(b, &ss); chk.E(err) {
+		return
+	}
+	for _, v := range ss {
+		var bb []byte
+		if bb, err = json.Marshal(&v); chk.E(err) {
+			return
+		}
+		u := &U{}
+		if err = json.Unmarshal(bb, u); chk.E(err) {
+			return
+		}
+		s.Ts = append(s.Ts, u)
+	}
+	return
+}
--- a/tag/tags_test.go
+++ b/tag/tags_test.go
@@ -1,8 +1,8 @@
 package tag

 import (
+	"bytes"
 	"encoding/json"
-	"net/url"
 	"testing"

 	"github.com/mleku/transit/chk"
@@ -10,22 +10,32 @@ import (
 )

 func TestNewTags(t *testing.T) {
-	root := T{"root", generateId()}
-	parent := T{"parent", generateId()}
-	eid := T{"event", generateId()}
-	pk := T{"user", generatePubkey()}
-	hashtag := "winning"
-	ht := T{"hashtag", hashtag}
-	var u *url.URL
 	var err error
-	if u, err = url.Parse("http://example.com"); chk.E(err) {
-		t.Fatal(err)
-	}
-	ut := T{"url", u}
+	root := New("root", generateId())
+	parent := New("parent", generateId())
+	eid := New("event", generateId())
+	pk := New("user", generatePubkey())
+	hashtag := "winning"
+	ht := New("hashtag", hashtag)
+	u := "http://example.com"
+	ut := New("url", u)
 	s := NewTags(root, parent, eid, pk, ht, ut)
+	// log.I.S(s)
 	var b []byte
 	if b, err = json.Marshal(s); chk.E(err) {
 		t.Fatal(err)
 	}
 	log.I.F("%s", b)
+	s2 := &S{}
+	if err = json.Unmarshal(b, s2); chk.E(err) {
+		t.Fatal(err)
+	}
+	var b2 []byte
+	if b2, err = json.Marshal(s2); chk.E(err) {
+		t.Fatal(err)
+	}
+	log.I.F("%s", b2)
+	if !bytes.Equal(b, b2) {
+		t.Fatal("failed to marshal/unmarshal")
+	}
 }
--- a/text/doc.go
+++ b/text/doc.go
@@ -0,0 +1,3 @@
+// Package text is a collection of helpers for working with text inside nostr
+// events such as implementing the string escaping scheme defined in NIP-01.
+package text
--- a/text/escape.go
+++ b/text/escape.go
@@ -0,0 +1,128 @@
+package text
+
+// Escape for JSON encoding according to RFC8259.
+//
+// This is the efficient implementation based on the NIP-01 specification:
+//
+// To prevent implementation differences from creating a different event Id for
+// the same event, the following rules MUST be followed while serializing:
+//
+//	No whitespace, line breaks or other unnecessary formatting should be included
+//	in the output JSON. No characters except the following should be escaped, and
+//	instead should be included verbatim:
+//
+//	- A line break, 0x0A, as \n
+//	- A double quote, 0x22, as \"
+//	- A backslash, 0x5C, as \\
+//	- A carriage return, 0x0D, as \r
+//	- A tab character, 0x09, as \t
+//	- A backspace, 0x08, as \b
+//	- A form feed, 0x0C, as \f
+//
+//	UTF-8 should be used for encoding.
+func Escape[V string | []byte](dstG, src V) []byte {
+	dst := []byte(dstG)
+	l := len(src)
+	for i := 0; i < l; i++ {
+		c := src[i]
+		switch {
+		case c == '"':
+			dst = append(dst, '\\', '"')
+		case c == '\\':
+			// if i+1 < l && src[i+1] == 'u' || i+1 < l && src[i+1] == '/' {
+			if i+1 < l && src[i+1] == 'u' {
+				dst = append(dst, '\\')
+			} else {
+				dst = append(dst, '\\', '\\')
+			}
+		case c == '\b':
+			dst = append(dst, '\\', 'b')
+		case c == '\t':
+			dst = append(dst, '\\', 't')
+		case c == '\n':
+			dst = append(dst, '\\', 'n')
+		case c == '\f':
+			dst = append(dst, '\\', 'f')
+		case c == '\r':
+			dst = append(dst, '\\', 'r')
+		default:
+			dst = append(dst, c)
+		}
+	}
+	return dst
+}
+
+// Unescape reverses the operation of Escape except instead of
+// appending it to the provided slice, it rewrites it, eliminating a memory
+// copy. Keep in mind that the original JSON will be mangled by this operation,
+// but the resultant slices will cost zero allocations.
+func Unescape(dst []byte) (b []byte) {
+	var r, w int
+	for ; r < len(dst); r++ {
+		if dst[r] == '\\' {
+			r++
+			c := dst[r]
+			switch {
+
+			// nip-01 specifies the following single letter C-style escapes for control
+			// codes under 0x20.
+			//
+			// no others are specified but must be preserved, so only these can be
+			// safely decoded at runtime as they must be re-encoded when marshalled.
+			case c == '"':
+				dst[w] = '"'
+				w++
+			case c == '\\':
+				dst[w] = '\\'
+				w++
+			case c == 'b':
+				dst[w] = '\b'
+				w++
+			case c == 't':
+				dst[w] = '\t'
+				w++
+			case c == 'n':
+				dst[w] = '\n'
+				w++
+			case c == 'f':
+				dst[w] = '\f'
+				w++
+			case c == 'r':
+				dst[w] = '\r'
+				w++
+
+				// special cases for non-nip-01 specified json escapes (must be preserved for Id
+				// generation).
+			case c == 'u':
+				dst[w] = '\\'
+				w++
+				dst[w] = 'u'
+				w++
+			case c == '/':
+				dst[w] = '\\'
+				w++
+				dst[w] = '/'
+				w++
+
+			// special case for octal escapes (must be preserved for Id generation).
+			case c >= '0' && c <= '9':
+				dst[w] = '\\'
+				w++
+				dst[w] = c
+				w++
+
+				// anything else after a reverse solidus just preserve it.
+			default:
+				dst[w] = dst[r]
+				w++
+				dst[w] = c
+				w++
+			}
+		} else {
+			dst[w] = dst[r]
+			w++
+		}
+	}
+	b = dst[:w]
+	return
+}
--- a/text/escape_test.go
+++ b/text/escape_test.go
@@ -0,0 +1,404 @@
+package text
+
+import (
+	"testing"
+
+	"lukechampine.com/frand"
+
+	"github.com/minio/sha256-simd"
+
+	"github.com/mleku/transit/chk"
+)
+
+func TestUnescapeByteString(t *testing.T) {
+	b := make([]byte, 256)
+	for i := range b {
+		b[i] = byte(i)
+	}
+	escaped := Escape(nil, b)
+	unescaped := Unescape(escaped)
+	if string(b) != string(unescaped) {
+		t.Log(b)
+		t.Log(unescaped)
+		t.FailNow()
+	}
+}
+
+func GenRandString(l int, src *frand.RNG) (str []byte) {
+	return src.Bytes(l)
+}
+
+var seed = sha256.Sum256([]byte(`
+The tao that can be told
+is not the eternal Tao
+The name that can be named
+is not the eternal Name
+
+The unnamable is the eternally real
+Naming is the origin of all particular things
+
+Free from desire, you realize the mystery
+Caught in desire, you see only the manifestations
+
+Yet mystery and manifestations arise from the same source
+This source is called darkness
+
+Darkness within darkness
+The gateway to all understanding
+`))
+
+var src = frand.NewCustom(seed[:], 32, 12)
+
+func TestRandomEscapeByteString(t *testing.T) {
+	// this is a kind of fuzz test, does a massive number of iterations of
+	// random content that ensures the escaping is correct without creating a
+	// fixed set of test vectors.
+
+	for i := 0; i < 1000; i++ {
+		l := src.Intn(1<<8) + 32
+		s1 := GenRandString(l, src)
+		s2 := make([]byte, l)
+		orig := make([]byte, l)
+		copy(s2, s1)
+		copy(orig, s1)
+
+		// first we are checking our implementation comports to the one from go-nostr.
+		escapeStringVersion := Escape([]byte{}, s1)
+		escapeJSONStringAndWrapVersion := Escape(nil, s2)
+		if len(escapeJSONStringAndWrapVersion) != len(escapeStringVersion) {
+			t.Logf("escapeString\nlength: %d\n%s\n%v\n",
+				len(escapeStringVersion), string(escapeStringVersion),
+				escapeStringVersion)
+			t.Logf("escapJSONStringAndWrap\nlength: %d\n%s\n%v\n",
+				len(escapeJSONStringAndWrapVersion),
+				escapeJSONStringAndWrapVersion,
+				escapeJSONStringAndWrapVersion)
+			t.FailNow()
+		}
+		for i := range escapeStringVersion {
+			if i > len(escapeJSONStringAndWrapVersion) {
+				t.Fatal("escapeString version is shorter")
+			}
+			if escapeStringVersion[i] != escapeJSONStringAndWrapVersion[i] {
+				t.Logf("escapeString version differs at index %d from "+
+					"escapeJSONStringAndWrap version\n%s\n%s\n%v\n%v", i,
+					escapeStringVersion[i-4:],
+					escapeJSONStringAndWrapVersion[i-4:],
+					escapeStringVersion[i-4:],
+					escapeJSONStringAndWrapVersion[i-4:])
+				t.Logf("escapeString\nlength: %d %s\n",
+					len(escapeStringVersion), escapeStringVersion)
+				t.Logf("escapJSONStringAndWrap\nlength: %d %s\n",
+					len(escapeJSONStringAndWrapVersion),
+					escapeJSONStringAndWrapVersion)
+				t.Logf("got '%s' %d expected '%s' %d\n",
+					string(escapeJSONStringAndWrapVersion[i]),
+					escapeJSONStringAndWrapVersion[i],
+					string(escapeStringVersion[i]),
+					escapeStringVersion[i],
+				)
+				t.FailNow()
+			}
+		}
+
+		// next, unescape the output and see if it matches the original
+		unescaped := Unescape(escapeJSONStringAndWrapVersion)
+		// t.Logf("unescaped: \n%s\noriginal:  \n%s", unescaped, orig)
+		if string(unescaped) != string(orig) {
+			t.Fatalf("\ngot      %d %v\nexpected %d %v\n",
+				len(unescaped),
+				unescaped,
+				len(orig),
+				orig,
+			)
+		}
+	}
+}
+
+func BenchmarkNostrEscapeNostrUnescape(b *testing.B) {
+	const size = 65536
+	b.Run("frand64k", func(b *testing.B) {
+		b.ReportAllocs()
+		in := make([]byte, size)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("NostrEscape64k", func(b *testing.B) {
+		b.ReportAllocs()
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			out = out[:0]
+		}
+	})
+	b.Run("NostrEscapeNostrUnescape64k", func(b *testing.B) {
+		b.ReportAllocs()
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			in = in[:0]
+			out = Unescape(out)
+			out = out[:0]
+		}
+	})
+	b.Run("frand32k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 2
+		in := make([]byte, size)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("NostrEscape32k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 2
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			out = out[:0]
+		}
+	})
+	b.Run("NostrEscapeNostrUnescape32k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 2
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			in = in[:0]
+			out = Unescape(out)
+			out = out[:0]
+		}
+	})
+	b.Run("frand16k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 4
+		in := make([]byte, size)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("NostrEscape16k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 4
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			out = out[:0]
+		}
+	})
+	b.Run("NostrEscapeNostrUnescape16k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 4
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			in = in[:0]
+			out = Unescape(out)
+			out = out[:0]
+		}
+	})
+	b.Run("frand8k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 8
+		in := make([]byte, size)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("NostrEscape8k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 8
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			out = out[:0]
+		}
+	})
+	b.Run("NostrEscapeNostrUnescape8k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 8
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			in = in[:0]
+			out = Unescape(out)
+			out = out[:0]
+		}
+	})
+	b.Run("frand4k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 16
+		in := make([]byte, size)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("NostrEscape4k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 16
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			out = out[:0]
+		}
+	})
+	b.Run("NostrEscapeNostrUnescape4k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 16
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			in = in[:0]
+			out = Unescape(out)
+			out = out[:0]
+		}
+	})
+	b.Run("frand2k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 32
+		in := make([]byte, size)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("NostrEscape2k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 32
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			out = out[:0]
+		}
+	})
+	b.Run("NostrEscapeNostrUnescape2k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 32
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			in = in[:0]
+			out = Unescape(out)
+			out = out[:0]
+		}
+	})
+	b.Run("frand1k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 64
+		in := make([]byte, size)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("NostrEscape1k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 64
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			out = out[:0]
+		}
+	})
+	b.Run("NostrEscapeNostrUnescape1k", func(b *testing.B) {
+		b.ReportAllocs()
+		size := size / 64
+		in := make([]byte, size)
+		out := make([]byte, size*2)
+		var err error
+		for i := 0; i < b.N; i++ {
+			if _, err = frand.Read(in); chk.E(err) {
+				b.Fatal(err)
+			}
+			out = Escape(out, in)
+			in = in[:0]
+			out = Unescape(out)
+			out = out[:0]
+		}
+	})
+}
--- a/text/helpers.go
+++ b/text/helpers.go
@@ -0,0 +1,248 @@
+package text
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/templexxx/xhex"
+
+	"github.com/mleku/transit/chk"
+	"github.com/mleku/transit/errorf"
+	"github.com/mleku/transit/hex"
+)
+
+// JSONKey generates the JSON format for an object key and terminates with the semicolon.
+func JSONKey(dst, k []byte) (b []byte) {
+	dst = append(dst, '"')
+	dst = append(dst, k...)
+	dst = append(dst, '"', ':')
+	b = dst
+	return
+}
+
+// UnmarshalHex takes a byte string that should contain a quoted hexadecimal encoded value,
+// decodes it in-place using a SIMD hex codec and returns the decoded truncated bytes (the other
+// half will be as it was but no allocation is required).
+func UnmarshalHex(b []byte) (h []byte, rem []byte, err error) {
+	rem = b[:]
+	var inQuote bool
+	var start int
+	for i := 0; i < len(b); i++ {
+		if !inQuote {
+			if b[i] == '"' {
+				inQuote = true
+				start = i + 1
+			}
+		} else if b[i] == '"' {
+			h = b[start:i]
+			rem = b[i+1:]
+			break
+		}
+	}
+	if !inQuote {
+		err = io.EOF
+		return
+	}
+	l := len(h)
+	if l%2 != 0 {
+		err = errorf.E("invalid length for hex: %d, %0x", len(h), h)
+		return
+	}
+	if err = xhex.Decode(h, h); chk.E(err) {
+		return
+	}
+	h = h[:l/2]
+	return
+}
+
+// UnmarshalQuoted performs an in-place unquoting of NIP-01 quoted byte string.
+func UnmarshalQuoted(b []byte) (content, rem []byte, err error) {
+	if len(b) == 0 {
+		err = io.EOF
+		return
+	}
+	rem = b[:]
+	for ; len(rem) >= 0; rem = rem[1:] {
+		// advance to open quotes
+		if rem[0] == '"' {
+			rem = rem[1:]
+			content = rem
+			break
+		}
+	}
+	if len(rem) == 0 {
+		err = io.EOF
+		return
+	}
+	var escaping bool
+	var contentLen int
+	for len(rem) > 0 {
+		if rem[0] == '\\' {
+			if !escaping {
+				escaping = true
+				contentLen++
+				rem = rem[1:]
+			} else {
+				escaping = false
+				contentLen++
+				rem = rem[1:]
+			}
+		} else if rem[0] == '"' {
+			if !escaping {
+				rem = rem[1:]
+				content = content[:contentLen]
+				content = Unescape(content)
+				return
+			}
+			contentLen++
+			rem = rem[1:]
+			escaping = false
+		} else {
+			escaping = false
+			switch rem[0] {
+			// none of these characters are allowed inside a JSON string:
+			//
+			// backspace, tab, newline, form feed or carriage return.
+			case '\b', '\t', '\n', '\f', '\r':
+				err = errorf.E("invalid character '%s' in quoted string",
+					Escape(nil, rem[:1]))
+				return
+			}
+			contentLen++
+			rem = rem[1:]
+		}
+	}
+	return
+}
+
+func MarshalHexArray(dst []byte, ha [][]byte) (b []byte) {
+	dst = append(dst, '[')
+	for i := range ha {
+		dst = AppendQuote(dst, ha[i], hex.EncAppend)
+		if i != len(ha)-1 {
+			dst = append(dst, ',')
+		}
+	}
+	dst = append(dst, ']')
+	b = dst
+	return
+}
+
+// UnmarshalHexArray unpacks a JSON array containing strings with hexadecimal, and checks all
+// values have the specified byte size.
+func UnmarshalHexArray(b []byte, size int) (t [][]byte, rem []byte, err error) {
+	rem = b
+	var openBracket bool
+	for ; len(rem) > 0; rem = rem[1:] {
+		if rem[0] == '[' {
+			openBracket = true
+		} else if openBracket {
+			if rem[0] == ',' {
+				continue
+			} else if rem[0] == ']' {
+				rem = rem[1:]
+				return
+			} else if rem[0] == '"' {
+				var h []byte
+				if h, rem, err = UnmarshalHex(rem); chk.E(err) {
+					return
+				}
+				if len(h) != size {
+					err = errorf.E("invalid hex array size, got %d expect %d",
+						2*len(h), 2*size)
+					return
+				}
+				t = append(t, h)
+				if rem[0] == ']' {
+					rem = rem[1:]
+					// done
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+// UnmarshalStringArray unpacks a JSON array containing strings.
+func UnmarshalStringArray(b []byte) (t [][]byte, rem []byte, err error) {
+	rem = b
+	var openBracket bool
+	for ; len(rem) > 0; rem = rem[1:] {
+		if rem[0] == '[' {
+			openBracket = true
+		} else if openBracket {
+			if rem[0] == ',' {
+				continue
+			} else if rem[0] == ']' {
+				rem = rem[1:]
+				return
+			} else if rem[0] == '"' {
+				var h []byte
+				if h, rem, err = UnmarshalQuoted(rem); chk.E(err) {
+					return
+				}
+				t = append(t, h)
+				if rem[0] == ']' {
+					rem = rem[1:]
+					// done
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+func True() []byte  { return []byte("true") }
+func False() []byte { return []byte("false") }
+
+func MarshalBool(src []byte, truth bool) []byte {
+	if truth {
+		return append(src, True()...)
+	}
+	return append(src, False()...)
+}
+
+func UnmarshalBool(src []byte) (rem []byte, truth bool, err error) {
+	rem = src
+	t, f := True(), False()
+	for i := range rem {
+		if rem[i] == t[0] {
+			if len(rem) < i+len(t) {
+				err = io.EOF
+				return
+			}
+			if bytes.Equal(t, rem[i:i+len(t)]) {
+				truth = true
+				rem = rem[i+len(t):]
+				return
+			}
+		}
+		if rem[i] == f[0] {
+			if len(rem) < i+len(f) {
+				err = io.EOF
+				return
+			}
+			if bytes.Equal(f, rem[i:i+len(f)]) {
+				rem = rem[i+len(f):]
+				return
+			}
+		}
+	}
+	// if a truth value is not found in the string it will run to the end
+	err = io.EOF
+	return
+}
+
+func Comma(b []byte) (rem []byte, err error) {
+	rem = b
+	for i := range rem {
+		if rem[i] == ',' {
+			rem = rem[i:]
+			return
+		}
+	}
+	err = io.EOF
+	return
+}
--- a/text/helpers_test.go
+++ b/text/helpers_test.go
@@ -0,0 +1,52 @@
+package text
+
+import (
+	"bytes"
+	"testing"
+
+	"lukechampine.com/frand"
+
+	"github.com/minio/sha256-simd"
+
+	"github.com/mleku/transit/chk"
+	"github.com/mleku/transit/hex"
+)
+
+func TestUnmarshalHexArray(t *testing.T) {
+	var ha [][]byte
+	h := make([]byte, sha256.Size)
+	frand.Read(h)
+	var dst []byte
+	for _ = range 20 {
+		hh := sha256.Sum256(h)
+		h = hh[:]
+		ha = append(ha, h)
+	}
+	dst = append(dst, '[')
+	for i := range ha {
+		dst = AppendQuote(dst, ha[i], hex.EncAppend)
+		if i != len(ha)-1 {
+			dst = append(dst, ',')
+		}
+	}
+	dst = append(dst, ']')
+	var ha2 [][]byte
+	var rem []byte
+	var err error
+	if ha2, rem, err = UnmarshalHexArray(dst, 32); chk.E(err) {
+		t.Fatal(err)
+	}
+	if len(ha2) != len(ha) {
+		t.Fatalf("failed to unmarshal, got %d fields, expected %d", len(ha2),
+			len(ha))
+	}
+	if len(rem) > 0 {
+		t.Fatalf("failed to unmarshal, remnant afterwards '%s'", rem)
+	}
+	for i := range ha2 {
+		if !bytes.Equal(ha[i], ha2[i]) {
+			t.Fatalf("failed to unmarshal at element %d; got %x, expected %x",
+				i, ha[i], ha2[i])
+		}
+	}
+}
--- a/text/hex.go
+++ b/text/hex.go
@@ -0,0 +1,34 @@
+package text
+
+import (
+	"github.com/mleku/transit/chk"
+	"github.com/mleku/transit/hex"
+)
+
+// AppendHexFromBinary appends to a hex output from binary input.
+func AppendHexFromBinary(dst, src []byte, quote bool) (b []byte) {
+	if quote {
+		dst = AppendQuote(dst, src, hex.EncAppend)
+	} else {
+		dst = hex.EncAppend(dst, src)
+	}
+	b = dst
+	return
+}
+
+// AppendBinaryFromHex encodes binary input as hex and appends it to the output.
+func AppendBinaryFromHex(dst, src []byte, unquote bool) (b []byte,
+	err error) {
+	if unquote {
+		if dst, err = hex.DecAppend(dst,
+			Unquote(src)); chk.E(err) {
+			return
+		}
+	} else {
+		if dst, err = hex.DecAppend(dst, src); chk.E(err) {
+			return
+		}
+	}
+	b = dst
+	return
+}
--- a/text/wrap.go
+++ b/text/wrap.go
@@ -0,0 +1,86 @@
+package text
+
+// AppendBytesClosure is a function type for appending data from a source to a destination and
+// returning the appended-to slice.
+type AppendBytesClosure func(dst, src []byte) []byte
+
+// AppendClosure is a simple append where the caller appends to the destination and returns the
+// appended-to slice.
+type AppendClosure func(dst []byte) []byte
+
+// Unquote removes the quotes around a slice of bytes.
+func Unquote(b []byte) []byte { return b[1 : len(b)-1] }
+
+// Noop simply appends the source to the destination slice and returns it.
+func Noop(dst, src []byte) []byte { return append(dst, src...) }
+
+// AppendQuote appends a source of bytes, that have been processed by an AppendBytesClosure and
+// returns the appended-to slice.
+func AppendQuote(dst, src []byte, ac AppendBytesClosure) []byte {
+	dst = append(dst, '"')
+	dst = ac(dst, src)
+	dst = append(dst, '"')
+	return dst
+}
+
+// Quote simply quotes a provided source and attaches it to the provided destination slice.
+func Quote(dst, src []byte) []byte { return AppendQuote(dst, src, Noop) }
+
+// AppendSingleQuote appends a provided AppendBytesClosure's output from a given source of
+// bytes, wrapped in single quotes ”.
+func AppendSingleQuote(dst, src []byte, ac AppendBytesClosure) []byte {
+	dst = append(dst, '\'')
+	dst = ac(dst, src)
+	dst = append(dst, '\'')
+	return dst
+}
+
+// AppendBackticks appends a provided AppendBytesClosure's output from a given source of
+// bytes, wrapped in backticks “.
+func AppendBackticks(dst, src []byte, ac AppendBytesClosure) []byte {
+	dst = append(dst, '`')
+	dst = ac(dst, src)
+	dst = append(dst, '`')
+	return dst
+}
+
+// AppendBrace appends a provided AppendBytesClosure's output from a given source of
+// bytes, wrapped in braces ().
+func AppendBrace(dst, src []byte, ac AppendBytesClosure) []byte {
+	dst = append(dst, '(')
+	dst = ac(dst, src)
+	dst = append(dst, ')')
+	return dst
+}
+
+// AppendParenthesis appends a provided AppendBytesClosure's output from a given source of
+// bytes, wrapped in parentheses {}.
+func AppendParenthesis(dst, src []byte, ac AppendBytesClosure) []byte {
+	dst = append(dst, '{')
+	dst = ac(dst, src)
+	dst = append(dst, '}')
+	return dst
+}
+
+// AppendBracket appends a provided AppendBytesClosure's output from a given source of
+// bytes, wrapped in brackets [].
+func AppendBracket(dst, src []byte, ac AppendBytesClosure) []byte {
+	dst = append(dst, '[')
+	dst = ac(dst, src)
+	dst = append(dst, ']')
+	return dst
+}
+
+// AppendList appends an input source bytes processed by an AppendBytesClosure and separates
+// elements with the given separator byte.
+func AppendList(dst []byte, src [][]byte, separator byte,
+	ac AppendBytesClosure) []byte {
+	last := len(src) - 1
+	for i := range src {
+		dst = append(dst, ac(dst, src[i])...)
+		if i < last {
+			dst = append(dst, separator)
+		}
+	}
+	return dst
+}