From bf178eae4e88bef92098c120f5e450335a6ee04d Mon Sep 17 00:00:00 2001 From: mleku Date: Fri, 22 Aug 2025 14:29:55 +0100 Subject: [PATCH] complete the marshal/unmarshal of events using the new pool enabled tag codecs --- app/handle-message.go | 5 + ...handleRelayinfo.go => handle-relayinfo.go} | 10 +- app/handle-websocket.go | 100 ++++++ app/helpers.go | 69 ++++ app/listener.go | 24 +- app/main.go | 2 +- app/server.go | 26 ++ cmd/eventpool/eventpool.go | 64 ++++ go.mod | 3 + go.sum | 6 + pkg/encoders/event/event.go | 322 +++++++++++++++++- pkg/encoders/event/event_test.go | 63 ++++ pkg/encoders/kind/kind.go | 18 +- pkg/encoders/tag/atag/atag.go | 50 +++ pkg/encoders/tag/atag/atag_test.go | 45 +++ pkg/encoders/tag/tag.go | 75 ++++ pkg/encoders/tag/tag_test.go | 32 ++ pkg/encoders/tag/tags.go | 78 +++++ pkg/encoders/tag/tags_test.go | 37 ++ pkg/encoders/text/escape.go | 16 +- pkg/encoders/text/helpers.go | 255 ++++++++++++++ pkg/encoders/text/helpers_test.go | 54 +++ pkg/encoders/text/wrap.go | 88 +++++ pkg/utils/bufpool/bufpool.go | 78 +++++ pkg/utils/bufpool/bufpool_test.go | 71 ++++ 25 files changed, 1547 insertions(+), 44 deletions(-) create mode 100644 app/handle-message.go rename app/{handleRelayinfo.go => handle-relayinfo.go} (87%) create mode 100644 app/handle-websocket.go create mode 100644 app/helpers.go create mode 100644 app/server.go create mode 100644 cmd/eventpool/eventpool.go create mode 100644 pkg/encoders/event/event_test.go create mode 100644 pkg/encoders/tag/atag/atag.go create mode 100644 pkg/encoders/tag/atag/atag_test.go create mode 100644 pkg/encoders/tag/tag.go create mode 100644 pkg/encoders/tag/tag_test.go create mode 100644 pkg/encoders/tag/tags.go create mode 100644 pkg/encoders/tag/tags_test.go create mode 100644 pkg/encoders/text/helpers.go create mode 100644 pkg/encoders/text/helpers_test.go create mode 100644 pkg/encoders/text/wrap.go create mode 100644 pkg/utils/bufpool/bufpool.go create mode 100644 pkg/utils/bufpool/bufpool_test.go diff --git a/app/handle-message.go b/app/handle-message.go new file mode 100644 index 0000000..7456322 --- /dev/null +++ b/app/handle-message.go @@ -0,0 +1,5 @@ +package app + +func (s *Server) HandleMessage() { + +} diff --git a/app/handleRelayinfo.go b/app/handle-relayinfo.go similarity index 87% rename from app/handleRelayinfo.go rename to app/handle-relayinfo.go index a341a1f..1739bc8 100644 --- a/app/handleRelayinfo.go +++ b/app/handle-relayinfo.go @@ -25,9 +25,9 @@ import ( // The function constructs a relay information document using either the // Informer interface implementation or predefined server configuration. It // returns this document as a JSON response to the client. -func (l *Listener) HandleRelayInfo(w http.ResponseWriter, r *http.Request) { +func (s *Server) HandleRelayInfo(w http.ResponseWriter, r *http.Request) { r.Header.Set("Content-Type", "application/json") - log.I.Ln("handling relay information document") + log.D.Ln("handling relay information document") var info *relayinfo.T supportedNIPs := relayinfo.GetList( relayinfo.BasicProtocol, @@ -47,14 +47,14 @@ func (l *Listener) HandleRelayInfo(w http.ResponseWriter, r *http.Request) { sort.Sort(supportedNIPs) log.T.Ln("supported NIPs", supportedNIPs) info = &relayinfo.T{ - Name: l.Config.AppName, + Name: s.Config.AppName, Description: version.Description, Nips: supportedNIPs, Software: version.URL, Version: version.V, Limitation: relayinfo.Limits{ - // AuthRequired: l.C.AuthRequired, - // RestrictedWrites: l.C.AuthRequired, + // AuthRequired: s.C.AuthRequired, + // RestrictedWrites: s.C.AuthRequired, }, Icon: "https://cdn.satellite.earth/ac9778868fbf23b63c47c769a74e163377e6ea94d3f0f31711931663d035c4f6.png", } diff --git a/app/handle-websocket.go b/app/handle-websocket.go new file mode 100644 index 0000000..009ce6c --- /dev/null +++ b/app/handle-websocket.go @@ -0,0 +1,100 @@ +package app + +import ( + "context" + "net/http" + "strings" + "time" + + "github.com/coder/websocket" + "lol.mleku.dev/chk" + "lol.mleku.dev/log" +) + +const ( + // CloseMessage denotes a close control message. The optional message + // payload contains a numeric code and text. Use the FormatCloseMessage + // function to format a close message payload. + CloseMessage = 8 + + // PingMessage denotes a ping control message. The optional message payload + // is UTF-8 encoded text. + PingMessage = 9 + + // PongMessage denotes a pong control message. The optional message payload + // is UTF-8 encoded text. + PongMessage = 10 +) + +func (s *Server) HandleWebsocket(w http.ResponseWriter, r *http.Request) { + remote := GetRemoteFromReq(r) + var cancel context.CancelFunc + s.Ctx, cancel = context.WithCancel(s.Ctx) + defer cancel() + var err error + var conn *websocket.Conn + if conn, err = websocket.Accept( + w, r, &websocket.AcceptOptions{}, + ); chk.E(err) { + return + } + defer conn.CloseNow() + + go s.Pinger(s.Ctx, conn, time.NewTicker(time.Second*10), cancel) + for { + select { + case <-s.Ctx.Done(): + return + default: + } + var typ websocket.MessageType + var message []byte + if typ, message, err = conn.Read(s.Ctx); err != nil { + if strings.Contains( + err.Error(), "use of closed network connection", + ) { + return + } + status := websocket.CloseStatus(err) + switch status { + case websocket.StatusNormalClosure, + websocket.StatusGoingAway, + websocket.StatusNoStatusRcvd, + websocket.StatusAbnormalClosure, + websocket.StatusProtocolError: + default: + log.E.F("unexpected close error from %s: %v", remote, err) + } + return + } + if typ == PingMessage { + if err = conn.Write(s.Ctx, PongMessage, message); chk.E(err) { + return + } + continue + } + go s.HandleMessage() + } +} + +func (s *Server) Pinger( + ctx context.Context, conn *websocket.Conn, ticker *time.Ticker, + cancel context.CancelFunc, +) { + defer func() { + cancel() + ticker.Stop() + }() + var err error + for { + select { + case <-ticker.C: + if err = conn.Write(ctx, PingMessage, nil); err != nil { + log.E.F("error writing ping: %v; closing websocket", err) + return + } + case <-ctx.Done(): + return + } + } +} diff --git a/app/helpers.go b/app/helpers.go new file mode 100644 index 0000000..c779e4c --- /dev/null +++ b/app/helpers.go @@ -0,0 +1,69 @@ +package app + +import ( + "net/http" + "strings" +) + +// GetRemoteFromReq retrieves the originating IP address of the client from +// an HTTP request, considering standard and non-standard proxy headers. +// +// # Parameters +// +// - r: The HTTP request object containing details of the client and +// routing information. +// +// # Return Values +// +// - rr: A string value representing the IP address of the originating +// remote client. +// +// # Expected behaviour +// +// The function first checks for the standardized "Forwarded" header (RFC 7239) +// to identify the original client IP. If that isn't available, it falls back to +// the "X-Forwarded-For" header. If both headers are absent, it defaults to +// using the request's RemoteAddr. +// +// For the "Forwarded" header, it extracts the client IP from the "for" +// parameter. For the "X-Forwarded-For" header, if it contains one IP, it +// returns that. If it contains two IPs, it returns the second. +func GetRemoteFromReq(r *http.Request) (rr string) { + // First check for the standardized Forwarded header (RFC 7239) + forwarded := r.Header.Get("Forwarded") + if forwarded != "" { + // Parse the Forwarded header which can contain multiple parameters + // + // Format: + // + // Forwarded: by=;for=;host=;proto= + parts := strings.Split(forwarded, ";") + for _, part := range parts { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, "for=") { + // Extract the client IP from the "for" parameter + forValue := strings.TrimPrefix(part, "for=") + // Remove quotes if present + forValue = strings.Trim(forValue, "\"") + // Handle IPv6 addresses which are enclosed in square brackets + forValue = strings.Trim(forValue, "[]") + return forValue + } + } + } + // If the Forwarded header is not available or doesn't contain "for" + // parameter, fall back to X-Forwarded-For + rem := r.Header.Get("X-Forwarded-For") + if rem == "" { + rr = r.RemoteAddr + } else { + splitted := strings.Split(rem, " ") + if len(splitted) == 1 { + rr = splitted[0] + } + if len(splitted) == 2 { + rr = splitted[1] + } + } + return +} diff --git a/app/listener.go b/app/listener.go index 07a0d8a..39a7111 100644 --- a/app/listener.go +++ b/app/listener.go @@ -1,29 +1,9 @@ package app import ( - "net/http" - - "lol.mleku.dev/log" - "next.orly.dev/app/config" + "github.com/coder/websocket" ) type Listener struct { - mux *http.ServeMux - Config *config.C -} - -func (l *Listener) ServeHTTP(w http.ResponseWriter, r *http.Request) { - log.I.F("path %v header %v", r.URL, r.Header) - if r.Header.Get("Upgrade") == "websocket" { - l.HandleWebsocket(w, r) - } else if r.Header.Get("Accept") == "application/nostr+json" { - l.HandleRelayInfo(w, r) - } else { - http.Error(w, "Upgrade required", http.StatusUpgradeRequired) - } -} - -func (l *Listener) HandleWebsocket(w http.ResponseWriter, r *http.Request) { - log.I.F("websocket") - return + conn *websocket.Conn } diff --git a/app/main.go b/app/main.go index 8ec30c7..d4de2d6 100644 --- a/app/main.go +++ b/app/main.go @@ -19,7 +19,7 @@ func Run(ctx context.Context, cfg *config.C) (quit chan struct{}) { } }() // start listener - l := &Listener{ + l := &Server{ Config: cfg, } addr := fmt.Sprintf("%s:%d", cfg.Listen, cfg.Port) diff --git a/app/server.go b/app/server.go new file mode 100644 index 0000000..f0a892a --- /dev/null +++ b/app/server.go @@ -0,0 +1,26 @@ +package app + +import ( + "context" + "net/http" + + "lol.mleku.dev/log" + "next.orly.dev/app/config" +) + +type Server struct { + mux *http.ServeMux + Config *config.C + Ctx context.Context +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + log.T.F("path %v header %v", r.URL, r.Header) + if r.Header.Get("Upgrade") == "websocket" { + s.HandleWebsocket(w, r) + } else if r.Header.Get("Accept") == "application/nostr+json" { + s.HandleRelayInfo(w, r) + } else { + http.Error(w, "Upgrade required", http.StatusUpgradeRequired) + } +} diff --git a/cmd/eventpool/eventpool.go b/cmd/eventpool/eventpool.go new file mode 100644 index 0000000..bfd7079 --- /dev/null +++ b/cmd/eventpool/eventpool.go @@ -0,0 +1,64 @@ +package main + +import ( + "time" + + "github.com/pkg/profile" + lol "lol.mleku.dev" + "lol.mleku.dev/chk" + "lukechampine.com/frand" + "next.orly.dev/pkg/encoders/event" + "next.orly.dev/pkg/encoders/hex" + "next.orly.dev/pkg/encoders/tag" + "next.orly.dev/pkg/utils" + "next.orly.dev/pkg/utils/bufpool" +) + +func main() { + lol.SetLogLevel("info") + prof := profile.Start(profile.CPUProfile) + defer prof.Stop() + for range 1000000 { + ev := event.New() + ev.ID = frand.Bytes(32) + ev.Pubkey = frand.Bytes(32) + ev.CreatedAt = time.Now().Unix() + ev.Kind = 1 + ev.Tags = &tag.S{ + {T: [][]byte{[]byte("t"), []byte("hashtag")}}, + { + T: [][]byte{ + []byte("e"), + hex.EncAppend(nil, frand.Bytes(32)), + }, + }, + } + ev.Content = frand.Bytes(frand.Intn(1024) + 1) + ev.Sig = frand.Bytes(64) + // log.I.S(ev) + b, err := ev.MarshalJSON() + if chk.E(err) { + return + } + var bc []byte + bc = append(bc, b...) + // log.I.F("%s", bc) + ev2 := event.New() + if err = ev2.UnmarshalJSON(b); chk.E(err) { + return + } + var b2 []byte + if b2, err = ev.MarshalJSON(); err != nil { + return + } + if !utils.FastEqual(bc, b2) { + return + } + // free up the resources for the next iteration + ev.Free() + ev2.Free() + bufpool.PutBytes(b) + bufpool.PutBytes(b2) + bufpool.PutBytes(bc) + } +} diff --git a/go.mod b/go.mod index 0fc6770..37c14dd 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/adrg/xdg v0.5.3 + github.com/coder/websocket v1.8.13 github.com/davecgh/go-spew v1.1.1 github.com/klauspost/cpuid/v2 v2.3.0 github.com/pkg/profile v1.7.0 @@ -19,6 +20,8 @@ require ( github.com/fatih/color v1.18.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect + github.com/karrick/bufpool v1.2.0 // indirect + github.com/karrick/gopool v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index dad5f28..b6be032 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE= +github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -13,6 +15,10 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/karrick/bufpool v1.2.0 h1:AfhYmVv8A62iOzB31RuJrGLTdHlvBbl0+rh8Gvgvybg= +github.com/karrick/bufpool v1.2.0/go.mod h1:ZRBxSXJi05b7mfd7kcL1M86UL1x8dTValcwCQp7I7P8= +github.com/karrick/gopool v1.1.0 h1:b9C9zwnRjgu9RNQPfiGEFmCDm3OdRuLpY7qYIDf8b28= +github.com/karrick/gopool v1.1.0/go.mod h1:Llf0mwk3WWtY0AIQoodGWVOU+5xfvUWqJKvck2qNwBU= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= diff --git a/pkg/encoders/event/event.go b/pkg/encoders/event/event.go index 709abc6..14b1cea 100644 --- a/pkg/encoders/event/event.go +++ b/pkg/encoders/event/event.go @@ -1,7 +1,26 @@ package event +import ( + "fmt" + "io" + + "github.com/templexxx/xhex" + "lol.mleku.dev/chk" + "lol.mleku.dev/errorf" + "lol.mleku.dev/log" + "next.orly.dev/pkg/crypto/ec/schnorr" + "next.orly.dev/pkg/crypto/sha256" + "next.orly.dev/pkg/encoders/ints" + "next.orly.dev/pkg/encoders/kind" + "next.orly.dev/pkg/encoders/tag" + "next.orly.dev/pkg/encoders/text" + "next.orly.dev/pkg/utils" + "next.orly.dev/pkg/utils/bufpool" +) + // E is the primary datatype of nostr. This is the form of the structure that -// defines its JSON string-based format. +// defines its JSON string-based format. Always use New() and Free() to create +// and free event.E. type E struct { // ID is the SHA256 hash of the canonical encoding of the event in binary format @@ -19,7 +38,7 @@ type E struct { // Tags are a list of tags, which are a list of strings usually structured // as a 3-layer scheme indicating specific features of an event. - Tags [][]byte + Tags *tag.S // Content is an arbitrary string that can contain anything, but usually // conforming to a specification relating to the Kind and the Tags. @@ -28,6 +47,305 @@ type E struct { // Sig is the signature on the ID hash that validates as coming from the // Pubkey in binary format. Sig []byte + + // b is the decode buffer for the event.E. this is where the UnmarshalJSON will + // source the memory to store all of the fields except for the tags. + b bufpool.B +} + +var ( + jId = []byte("id") + jPubkey = []byte("pubkey") + jCreatedAt = []byte("created_at") + jKind = []byte("kind") + jTags = []byte("tags") + jContent = []byte("content") + jSig = []byte("sig") +) + +// New returns a new event.E. The returned event.E should be freed with Free() +// to return the unmarshalling buffer to the bufpool. +func New() *E { + return &E{ + b: bufpool.Get(), + } +} + +// Free returns the event.E to the pool, as well as nilling all of the fields. +// This should hint to the GC that the event.E can be freed, and the memory +// reused. The decode buffer will be returned to the pool for reuse. +func (ev *E) Free() { + bufpool.Put(ev.b) + ev.ID = nil + ev.Pubkey = nil + ev.Tags = nil + ev.Content = nil + ev.Sig = nil + ev.b = nil +} + +// MarshalJSON marshals an event.E into a JSON byte string. +// +// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use. +func (ev *E) MarshalJSON() (b []byte, err error) { + b = bufpool.Get() + b = b[:0] + b = append(b, '{') + b = append(b, '"') + b = append(b, jId...) + b = append(b, `":"`...) + b = b[:len(b)+2*sha256.Size] + xhex.Encode(b[len(b)-2*sha256.Size:], ev.ID) + b = append(b, `","`...) + b = append(b, jPubkey...) + b = append(b, `":"`...) + b = b[:len(b)+2*schnorr.PubKeyBytesLen] + xhex.Encode(b[len(b)-2*schnorr.PubKeyBytesLen:], ev.Pubkey) + b = append(b, `","`...) + b = append(b, jCreatedAt...) + b = append(b, `":`...) + b = ints.New(ev.CreatedAt).Marshal(b) + b = append(b, `,"`...) + b = append(b, jKind...) + b = append(b, `":`...) + b = ints.New(ev.Kind).Marshal(b) + b = append(b, `,"`...) + b = append(b, jTags...) + b = append(b, `":[`...) + lts := len(*ev.Tags) - 1 + for i, tt := range *ev.Tags { + b = append(b, '[') + lt := len(tt.T) - 1 + for j, t := range tt.T { + b = append(b, '"') + b = append(b, t...) + b = append(b, '"') + if j < lt { + b = append(b, ',') + } + } + b = append(b, ']') + if i < lts { + b = append(b, ',') + } + } + b = append(b, `],"`...) + b = append(b, jContent...) + b = append(b, `":"`...) + // it can happen the slice has insufficient capacity to hold the content AND + // the signature at this point, because the signature encoder must have + // sufficient capacity pre-allocated as it does not append to the buffer. + // unlike every other encoding function up to this point. + if cap(b) < len(b)+len(ev.Content)+7+256+2 { + b2 := make([]byte, len(b)+len(ev.Content)*2+7+256+2) + copy(b2, b) + b2 = b2[:len(b)] + // return the old buffer to the pool for reuse. + bufpool.PutBytes(b) + b = b2 + } + b = text.NostrEscape(b, ev.Content) + b = append(b, `","`...) + b = append(b, jSig...) + b = append(b, `":"`...) + b = b[:len(b)+2*schnorr.SignatureSize] + xhex.Encode(b[len(b)-2*schnorr.SignatureSize:], ev.Sig) + b = append(b, `"}`...) + return +} + +// UnmarshalJSON unmarshalls a JSON string into an event.E. +// +// Call ev.Free() to return the provided buffer to the bufpool afterwards. +func (ev *E) UnmarshalJSON(b []byte) (err error) { + key := make([]byte, 0, 9) + for ; len(b) > 0; b = b[1:] { + // Skip whitespace + if isWhitespace(b[0]) { + continue + } + if b[0] == '{' { + b = b[1:] + goto BetweenKeys + } + } + goto eof +BetweenKeys: + for ; len(b) > 0; b = b[1:] { + // Skip whitespace + if isWhitespace(b[0]) { + continue + } + if b[0] == '"' { + b = b[1:] + goto InKey + } + } + goto eof +InKey: + for ; len(b) > 0; b = b[1:] { + if b[0] == '"' { + b = b[1:] + goto InKV + } + key = append(key, b[0]) + } + goto eof +InKV: + for ; len(b) > 0; b = b[1:] { + // Skip whitespace + if isWhitespace(b[0]) { + continue + } + if b[0] == ':' { + b = b[1:] + goto InVal + } + } + goto eof +InVal: + // Skip whitespace before value + for len(b) > 0 && isWhitespace(b[0]) { + b = b[1:] + } + switch key[0] { + case jId[0]: + if !utils.FastEqual(jId, key) { + goto invalid + } + var id []byte + if id, b, err = text.UnmarshalHex(b); chk.E(err) { + return + } + if len(id) != sha256.Size { + err = errorf.E( + "invalid ID, require %d got %d", sha256.Size, + len(id), + ) + return + } + ev.ID = id + goto BetweenKV + case jPubkey[0]: + if !utils.FastEqual(jPubkey, key) { + goto invalid + } + var pk []byte + if pk, b, err = text.UnmarshalHex(b); chk.E(err) { + return + } + if len(pk) != schnorr.PubKeyBytesLen { + err = errorf.E( + "invalid pubkey, require %d got %d", + schnorr.PubKeyBytesLen, len(pk), + ) + return + } + ev.Pubkey = pk + goto BetweenKV + case jKind[0]: + if !utils.FastEqual(jKind, key) { + goto invalid + } + k := kind.New(0) + if b, err = k.Unmarshal(b); chk.E(err) { + return + } + ev.Kind = k.ToU16() + goto BetweenKV + case jTags[0]: + if !utils.FastEqual(jTags, key) { + goto invalid + } + ev.Tags = new(tag.S) + if b, err = ev.Tags.Unmarshal(b); chk.E(err) { + return + } + goto BetweenKV + case jSig[0]: + if !utils.FastEqual(jSig, key) { + goto invalid + } + var sig []byte + if sig, b, err = text.UnmarshalHex(b); chk.E(err) { + return + } + if len(sig) != schnorr.SignatureSize { + err = errorf.E( + "invalid sig length, require %d got %d '%s'\n%s", + schnorr.SignatureSize, len(sig), b, b, + ) + return + } + ev.Sig = sig + goto BetweenKV + case jContent[0]: + if key[1] == jContent[1] { + if !utils.FastEqual(jContent, key) { + goto invalid + } + if ev.Content, b, err = text.UnmarshalQuoted(b); chk.T(err) { + return + } + goto BetweenKV + } else if key[1] == jCreatedAt[1] { + if !utils.FastEqual(jCreatedAt, key) { + goto invalid + } + if b, err = ints.New(0).Unmarshal(b); chk.T(err) { + return + } + goto BetweenKV + } else { + goto invalid + } + default: + goto invalid + } +BetweenKV: + key = key[:0] + for ; len(b) > 0; b = b[1:] { + // Skip whitespace + if isWhitespace(b[0]) { + continue + } + + switch { + case len(b) == 0: + return + case b[0] == '}': + b = b[1:] + goto AfterClose + case b[0] == ',': + b = b[1:] + goto BetweenKeys + case b[0] == '"': + b = b[1:] + goto InKey + } + } + log.I.F("between kv") + goto eof +AfterClose: + // Skip any trailing whitespace + for len(b) > 0 && isWhitespace(b[0]) { + b = b[1:] + } + return +invalid: + err = fmt.Errorf( + "invalid key,\n'%s'\n'%s'\n'%s'", string(b), string(b[:len(b)]), + string(b), + ) + return +eof: + err = io.EOF + return +} + +// isWhitespace returns true if the byte is a whitespace character (space, tab, newline, carriage return). +func isWhitespace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' } // S is an array of event.E that sorts in reverse chronological order. diff --git a/pkg/encoders/event/event_test.go b/pkg/encoders/event/event_test.go new file mode 100644 index 0000000..d2a5abc --- /dev/null +++ b/pkg/encoders/event/event_test.go @@ -0,0 +1,63 @@ +package event + +import ( + "testing" + "time" + + "github.com/pkg/profile" + "lol.mleku.dev/chk" + "lukechampine.com/frand" + "next.orly.dev/pkg/encoders/hex" + "next.orly.dev/pkg/encoders/tag" + "next.orly.dev/pkg/utils" + "next.orly.dev/pkg/utils/bufpool" +) + +func TestMarshalJSON(t *testing.T) { + // lol.SetLogLevel("trace") + prof := profile.Start(profile.MemProfile) + defer prof.Stop() + for range 1000000 { + ev := New() + ev.ID = frand.Bytes(32) + ev.Pubkey = frand.Bytes(32) + ev.CreatedAt = time.Now().Unix() + ev.Kind = 1 + ev.Tags = &tag.S{ + {T: [][]byte{[]byte("t"), []byte("hashtag")}}, + { + T: [][]byte{ + []byte("e"), + hex.EncAppend(nil, frand.Bytes(32)), + }, + }, + } + ev.Content = frand.Bytes(frand.Intn(1024) + 1) + ev.Sig = frand.Bytes(64) + // log.I.S(ev) + b, err := ev.MarshalJSON() + if err != nil { + t.Fatal(err) + } + var bc []byte + bc = append(bc, b...) + // log.I.F("%s", bc) + ev2 := New() + if err = ev2.UnmarshalJSON(b); chk.E(err) { + t.Fatal(err) + } + var b2 []byte + if b2, err = ev.MarshalJSON(); err != nil { + t.Fatal(err) + } + if !utils.FastEqual(bc, b2) { + t.Errorf("failed to re-marshal back original") + } + // free up the resources for the next iteration + ev.Free() + ev2.Free() + bufpool.PutBytes(b) + bufpool.PutBytes(b2) + bufpool.PutBytes(bc) + } +} diff --git a/pkg/encoders/kind/kind.go b/pkg/encoders/kind/kind.go index 8c23313..bcb03ff 100644 --- a/pkg/encoders/kind/kind.go +++ b/pkg/encoders/kind/kind.go @@ -19,8 +19,8 @@ type K struct { K uint16 } -// New creates a new kind.K with a provided integer value. Note that anything larger than 2^16 -// will be truncated. +// New creates a new kind.K with a provided integer value. Note that anything +// larger than 2^16 will be truncated. func New[V constraints.Integer](k V) (ki *K) { return &K{uint16(k)} } // ToInt returns the value of the kind.K as an int. @@ -55,7 +55,8 @@ func (k *K) ToU64() uint64 { return uint64(k.K) } -// Name returns the human readable string describing the semantics of the kind.K. +// Name returns the human readable string describing the semantics of the +// kind.K. func (k *K) Name() string { return GetString(k) } // Equal checks if @@ -76,8 +77,8 @@ var Privileged = []*K{ PrivateDirectMessage, } -// IsPrivileged returns true if the type is the kind of message nobody else than the pubkeys in -// the event and p tags of the event are party to. +// IsPrivileged returns true if the type is the kind of message nobody else than +// the pubkeys in the event and p tags of the event are party to. func (k *K) IsPrivileged() (is bool) { for i := range Privileged { if k.Equal(Privileged[i]) { @@ -87,8 +88,11 @@ func (k *K) IsPrivileged() (is bool) { return } -// Marshal renders the kind.K into bytes containing the ASCII string form of the kind number. -func (k *K) Marshal(dst []byte) (b []byte) { return ints.New(k.ToU64()).Marshal(dst) } +// Marshal renders the kind.K into bytes containing the ASCII string form of the +// kind number. +func (k *K) Marshal(dst []byte) (b []byte) { + return ints.New(k.ToU64()).Marshal(dst) +} // Unmarshal decodes a byte string into a kind.K. func (k *K) Unmarshal(b []byte) (r []byte, err error) { diff --git a/pkg/encoders/tag/atag/atag.go b/pkg/encoders/tag/atag/atag.go new file mode 100644 index 0000000..e031200 --- /dev/null +++ b/pkg/encoders/tag/atag/atag.go @@ -0,0 +1,50 @@ +// Package atag implements a special, optimized handling for keeping a tags +// (address) in a more memory efficient form while working with these tags. +package atag + +import ( + "bytes" + + "lol.mleku.dev/chk" + "next.orly.dev/pkg/encoders/hex" + "next.orly.dev/pkg/encoders/ints" + "next.orly.dev/pkg/encoders/kind" +) + +// T is a data structure for what is found in an `a` tag: kind:pubkey:arbitrary data +type T struct { + Kind *kind.K + PubKey []byte + DTag []byte +} + +// Marshal an atag.T into raw bytes. +func (t *T) Marshal(dst []byte) (b []byte) { + b = t.Kind.Marshal(dst) + b = append(b, ':') + b = hex.EncAppend(b, t.PubKey) + b = append(b, ':') + b = append(b, t.DTag...) + return +} + +// Unmarshal an atag.T from its ascii encoding. +func (t *T) Unmarshal(b []byte) (r []byte, err error) { + split := bytes.Split(b, []byte{':'}) + if len(split) != 3 { + return + } + // kind + kin := ints.New(uint16(0)) + if _, err = kin.Unmarshal(split[0]); chk.E(err) { + return + } + t.Kind = kind.New(kin.Uint16()) + // pubkey + if t.PubKey, err = hex.DecAppend(t.PubKey, split[1]); chk.E(err) { + return + } + // d-tag + t.DTag = split[2] + return +} diff --git a/pkg/encoders/tag/atag/atag_test.go b/pkg/encoders/tag/atag/atag_test.go new file mode 100644 index 0000000..6227c39 --- /dev/null +++ b/pkg/encoders/tag/atag/atag_test.go @@ -0,0 +1,45 @@ +package atag + +import ( + "math" + "testing" + + "lol.mleku.dev/chk" + "lol.mleku.dev/log" + "lukechampine.com/frand" + "next.orly.dev/pkg/crypto/ec/schnorr" + "next.orly.dev/pkg/encoders/hex" + "next.orly.dev/pkg/encoders/kind" + "next.orly.dev/pkg/utils" +) + +func TestT_Marshal_Unmarshal(t *testing.T) { + k := kind.New(frand.Intn(math.MaxUint16)) + pk := make([]byte, schnorr.PubKeyBytesLen) + frand.Read(pk) + d := make([]byte, frand.Intn(10)+3) + frand.Read(d) + var dtag string + dtag = hex.Enc(d) + t1 := &T{ + Kind: k, + PubKey: pk, + DTag: []byte(dtag), + } + b1 := t1.Marshal(nil) + log.I.F("%s", b1) + t2 := &T{} + var r []byte + var err error + if r, err = t2.Unmarshal(b1); chk.E(err) { + t.Fatal(err) + } + if len(r) > 0 { + log.I.S(r) + t.Fatalf("remainder") + } + b2 := t2.Marshal(nil) + if !utils.FastEqual(b1, b2) { + t.Fatalf("failed to re-marshal back original") + } +} diff --git a/pkg/encoders/tag/tag.go b/pkg/encoders/tag/tag.go new file mode 100644 index 0000000..28512b6 --- /dev/null +++ b/pkg/encoders/tag/tag.go @@ -0,0 +1,75 @@ +// Package tag provides an implementation of a nostr tag list, an array of +// strings with a usually single letter first "key" field, including methods to +// compare, marshal/unmarshal and access elements with their proper semantics. +package tag + +import ( + "lol.mleku.dev/errorf" + "next.orly.dev/pkg/encoders/text" + "next.orly.dev/pkg/utils/bufpool" +) + +// The tag position meanings, so they are clear when reading. +const ( + Key = iota + Value + Relay +) + +type T struct { + T [][]byte + b bufpool.B +} + +func New(t ...[]byte) *T { + return &T{T: t, b: bufpool.Get()} +} + +func (t *T) Free() { + bufpool.Put(t.b) + t.T = nil +} + +// Marshal encodes a tag.T as standard minified JSON array of strings. +// +// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use. +func (t *T) Marshal() (b []byte) { + dst := t.b + dst = append(dst, '[') + for i, s := range t.T { + dst = text.AppendQuote(dst, s, text.NostrEscape) + if i < len(t.T)-1 { + dst = append(dst, ',') + } + } + dst = append(dst, ']') + return dst +} + +// Unmarshal decodes a standard minified JSON array of strings to a tags.T. +// +// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use. +func (t *T) Unmarshal(b []byte) (r []byte, err error) { + var inQuotes, openedBracket bool + var quoteStart int + for i := 0; i < len(b); i++ { + if !openedBracket && b[i] == '[' { + openedBracket = true + } else if !inQuotes { + if b[i] == '"' { + inQuotes, quoteStart = true, i+1 + } else if b[i] == ']' { + return b[i+1:], err + } + } else if b[i] == '\\' && i < len(b)-1 { + i++ + } else if b[i] == '"' { + inQuotes = false + t.T = append(t.T, text.NostrUnescape(b[quoteStart:i])) + } + } + if !openedBracket || inQuotes { + return nil, errorf.E("tag: failed to parse tag") + } + return +} diff --git a/pkg/encoders/tag/tag_test.go b/pkg/encoders/tag/tag_test.go new file mode 100644 index 0000000..b4e6802 --- /dev/null +++ b/pkg/encoders/tag/tag_test.go @@ -0,0 +1,32 @@ +package tag + +import ( + "testing" + + "lol.mleku.dev/chk" + "lukechampine.com/frand" + "next.orly.dev/pkg/utils" +) + +func TestMarshalUnmarshal(t *testing.T) { + for _ = range 1000 { + n := frand.Intn(8) + tg := New() + for _ = range n { + b1 := make([]byte, frand.Intn(8)) + _, _ = frand.Read(b1) + tg.T = append(tg.T, b1) + } + tb := tg.Marshal() + var tbc []byte + tbc = append(tbc, tb...) + tg2 := New() + if _, err := tg2.Unmarshal(tb); chk.E(err) { + t.Fatal(err) + } + tb2 := tg2.Marshal() + if !utils.FastEqual(tbc, tb2) { + t.Fatalf("failed to re-marshal back original") + } + } +} diff --git a/pkg/encoders/tag/tags.go b/pkg/encoders/tag/tags.go new file mode 100644 index 0000000..f5def4c --- /dev/null +++ b/pkg/encoders/tag/tags.go @@ -0,0 +1,78 @@ +package tag + +import ( + "lol.mleku.dev/chk" + "next.orly.dev/pkg/utils/bufpool" +) + +// S is a list of tag.T - which are lists of string elements with ordering and +// no uniqueness constraint (not a set). +type S []*T + +// MarshalJSON encodes a tags.T appended to a provided byte slice in JSON form. +// +// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use. +func (s *S) MarshalJSON() (b []byte, err error) { + b = bufpool.Get() + b = append(b, '[') + for i, ss := range *s { + b = append(b, ss.Marshal()...) + if i < len(*s)-1 { + b = append(b, ',') + } + } + b = append(b, ']') + return +} + +// UnmarshalJSON a tags.T from a provided byte slice and return what remains +// after the end of the array. +// +// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use. +func (s *S) UnmarshalJSON(b []byte) (err error) { + _, err = s.Unmarshal(b) + return +} + +// Unmarshal a tags.T from a provided byte slice and return what remains after +// the end of the array. +func (s *S) Unmarshal(b []byte) (r []byte, err error) { + r = b[:] + for len(r) > 0 { + switch r[0] { + case '[': + r = r[1:] + goto inTags + case ',': + r = r[1:] + // next + case ']': + r = r[1:] + // the end + return + default: + r = r[1:] + } + inTags: + for len(r) > 0 { + switch r[0] { + case '[': + tt := New() + if r, err = tt.Unmarshal(r); chk.E(err) { + return + } + *s = append(*s, tt) + case ',': + r = r[1:] + // next + case ']': + r = r[1:] + // the end + return + default: + r = r[1:] + } + } + } + return +} diff --git a/pkg/encoders/tag/tags_test.go b/pkg/encoders/tag/tags_test.go new file mode 100644 index 0000000..d15d644 --- /dev/null +++ b/pkg/encoders/tag/tags_test.go @@ -0,0 +1,37 @@ +package tag + +import ( + "testing" + + "lol.mleku.dev/chk" + "lukechampine.com/frand" + "next.orly.dev/pkg/utils" +) + +func TestSMarshalUnmarshal(t *testing.T) { + for _ = range 100 { + tgs := new(S) + n := frand.Intn(8) + for _ = range n { + n := frand.Intn(8) + tg := New() + for _ = range n { + b1 := make([]byte, frand.Intn(8)) + _, _ = frand.Read(b1) + tg.T = append(tg.T, b1) + } + *tgs = append(*tgs, tg) + } + tgsb, _ := tgs.MarshalJSON() + var tbc []byte + tbc = append(tbc, tgsb...) + tgs2 := new(S) + if err := tgs2.UnmarshalJSON(tgsb); chk.E(err) { + t.Fatal(err) + } + tgsb2, _ := tgs2.MarshalJSON() + if !utils.FastEqual(tbc, tgsb2) { + t.Fatalf("failed to re-marshal back original") + } + } +} diff --git a/pkg/encoders/text/escape.go b/pkg/encoders/text/escape.go index fa2915d..804fe2d 100644 --- a/pkg/encoders/text/escape.go +++ b/pkg/encoders/text/escape.go @@ -63,11 +63,12 @@ func NostrUnescape(dst []byte) (b []byte) { c := dst[r] switch { - // nip-01 specifies the following single letter C-style escapes for control - // codes under 0x20. + // nip-01 specifies the following single letter C-style escapes for + // control codes under 0x20. // - // no others are specified but must be preserved, so only these can be - // safely decoded at runtime as they must be re-encoded when marshalled. + // no others are specified but must be preserved, so only these can + // be safely decoded at runtime as they must be re-encoded when + // marshalled. case c == '"': dst[w] = '"' w++ @@ -90,8 +91,8 @@ func NostrUnescape(dst []byte) (b []byte) { dst[w] = '\r' w++ - // special cases for non-nip-01 specified json escapes (must be preserved for ID - // generation). + // special cases for non-nip-01 specified json escapes (must be + // preserved for ID generation). case c == 'u': dst[w] = '\\' w++ @@ -103,7 +104,8 @@ func NostrUnescape(dst []byte) (b []byte) { dst[w] = '/' w++ - // special case for octal escapes (must be preserved for ID generation). + // special case for octal escapes (must be preserved for ID + // generation). case c >= '0' && c <= '9': dst[w] = '\\' w++ diff --git a/pkg/encoders/text/helpers.go b/pkg/encoders/text/helpers.go new file mode 100644 index 0000000..ccfe395 --- /dev/null +++ b/pkg/encoders/text/helpers.go @@ -0,0 +1,255 @@ +package text + +import ( + "io" + + "github.com/templexxx/xhex" + "lol.mleku.dev/chk" + "lol.mleku.dev/errorf" + "next.orly.dev/pkg/encoders/hex" + "next.orly.dev/pkg/utils" +) + +// JSONKey generates the JSON format for an object key and terminates with the semicolon. +func JSONKey(dst, k []byte) (b []byte) { + dst = append(dst, '"') + dst = append(dst, k...) + dst = append(dst, '"', ':') + b = dst + return +} + +// UnmarshalHex takes a byte string that should contain a quoted hexadecimal +// encoded value, decodes it using a SIMD hex codec and returns the decoded +// bytes in a newly allocated buffer. +func UnmarshalHex(b []byte) (h []byte, rem []byte, err error) { + rem = b[:] + var inQuote bool + var start int + for i := 0; i < len(b); i++ { + if !inQuote { + if b[i] == '"' { + inQuote = true + start = i + 1 + } + } else if b[i] == '"' { + hexStr := b[start:i] + rem = b[i+1:] + l := len(hexStr) + if l%2 != 0 { + err = errorf.E( + "invalid length for hex: %d, %0x", + len(hexStr), hexStr, + ) + return + } + // Allocate a new buffer for the decoded data + h = make([]byte, l/2) + if err = xhex.Decode(h, hexStr); chk.E(err) { + return + } + return + } + } + if !inQuote { + err = io.EOF + return + } + return +} + +// UnmarshalQuoted performs an in-place unquoting of NIP-01 quoted byte string. +func UnmarshalQuoted(b []byte) (content, rem []byte, err error) { + if len(b) == 0 { + err = io.EOF + return + } + rem = b[:] + for ; len(rem) >= 0; rem = rem[1:] { + // advance to open quotes + if rem[0] == '"' { + rem = rem[1:] + content = rem + break + } + } + if len(rem) == 0 { + err = io.EOF + return + } + var escaping bool + var contentLen int + for len(rem) > 0 { + if rem[0] == '\\' { + if !escaping { + escaping = true + contentLen++ + rem = rem[1:] + } else { + escaping = false + contentLen++ + rem = rem[1:] + } + } else if rem[0] == '"' { + if !escaping { + rem = rem[1:] + content = content[:contentLen] + content = NostrUnescape(content) + return + } + contentLen++ + rem = rem[1:] + escaping = false + } else { + escaping = false + switch rem[0] { + // none of these characters are allowed inside a JSON string: + // + // backspace, tab, newline, form feed or carriage return. + case '\b', '\t', '\n', '\f', '\r': + err = errorf.E( + "invalid character '%s' in quoted string", + NostrEscape(nil, rem[:1]), + ) + return + } + contentLen++ + rem = rem[1:] + } + } + return +} + +func MarshalHexArray(dst []byte, ha [][]byte) (b []byte) { + dst = append(dst, '[') + for i := range ha { + dst = AppendQuote(dst, ha[i], hex.EncAppend) + if i != len(ha)-1 { + dst = append(dst, ',') + } + } + dst = append(dst, ']') + b = dst + return +} + +// UnmarshalHexArray unpacks a JSON array containing strings with hexadecimal, and checks all +// values have the specified byte size. +func UnmarshalHexArray(b []byte, size int) (t [][]byte, rem []byte, err error) { + rem = b + var openBracket bool + for ; len(rem) > 0; rem = rem[1:] { + if rem[0] == '[' { + openBracket = true + } else if openBracket { + if rem[0] == ',' { + continue + } else if rem[0] == ']' { + rem = rem[1:] + return + } else if rem[0] == '"' { + var h []byte + if h, rem, err = UnmarshalHex(rem); chk.E(err) { + return + } + if len(h) != size { + err = errorf.E( + "invalid hex array size, got %d expect %d", + 2*len(h), 2*size, + ) + return + } + t = append(t, h) + if rem[0] == ']' { + rem = rem[1:] + // done + return + } + } + } + } + return +} + +// UnmarshalStringArray unpacks a JSON array containing strings. +func UnmarshalStringArray(b []byte) (t [][]byte, rem []byte, err error) { + rem = b + var openBracket bool + for ; len(rem) > 0; rem = rem[1:] { + if rem[0] == '[' { + openBracket = true + } else if openBracket { + if rem[0] == ',' { + continue + } else if rem[0] == ']' { + rem = rem[1:] + return + } else if rem[0] == '"' { + var h []byte + if h, rem, err = UnmarshalQuoted(rem); chk.E(err) { + return + } + t = append(t, h) + if rem[0] == ']' { + rem = rem[1:] + // done + return + } + } + } + } + return +} + +func True() []byte { return []byte("true") } +func False() []byte { return []byte("false") } + +func MarshalBool(src []byte, truth bool) []byte { + if truth { + return append(src, True()...) + } + return append(src, False()...) +} + +func UnmarshalBool(src []byte) (rem []byte, truth bool, err error) { + rem = src + t, f := True(), False() + for i := range rem { + if rem[i] == t[0] { + if len(rem) < i+len(t) { + err = io.EOF + return + } + if utils.FastEqual(t, rem[i:i+len(t)]) { + truth = true + rem = rem[i+len(t):] + return + } + } + if rem[i] == f[0] { + if len(rem) < i+len(f) { + err = io.EOF + return + } + if utils.FastEqual(f, rem[i:i+len(f)]) { + rem = rem[i+len(f):] + return + } + } + } + // if a truth value is not found in the string it will run to the end + err = io.EOF + return +} + +func Comma(b []byte) (rem []byte, err error) { + rem = b + for i := range rem { + if rem[i] == ',' { + rem = rem[i:] + return + } + } + err = io.EOF + return +} diff --git a/pkg/encoders/text/helpers_test.go b/pkg/encoders/text/helpers_test.go new file mode 100644 index 0000000..9bb15c9 --- /dev/null +++ b/pkg/encoders/text/helpers_test.go @@ -0,0 +1,54 @@ +package text + +import ( + "testing" + + "lol.mleku.dev/chk" + "lukechampine.com/frand" + "next.orly.dev/pkg/crypto/sha256" + "next.orly.dev/pkg/encoders/hex" + "next.orly.dev/pkg/utils" +) + +func TestUnmarshalHexArray(t *testing.T) { + var ha [][]byte + h := make([]byte, sha256.Size) + frand.Read(h) + var dst []byte + for _ = range 20 { + hh := sha256.Sum256(h) + h = hh[:] + ha = append(ha, h) + } + dst = append(dst, '[') + for i := range ha { + dst = AppendQuote(dst, ha[i], hex.EncAppend) + if i != len(ha)-1 { + dst = append(dst, ',') + } + } + dst = append(dst, ']') + var ha2 [][]byte + var rem []byte + var err error + if ha2, rem, err = UnmarshalHexArray(dst, sha256.Size); chk.E(err) { + t.Fatal(err) + } + if len(ha2) != len(ha) { + t.Fatalf( + "failed to unmarshal, got %d fields, expected %d", len(ha2), + len(ha), + ) + } + if len(rem) > 0 { + t.Fatalf("failed to unmarshal, remnant afterwards '%s'", rem) + } + for i := range ha2 { + if !utils.FastEqual(ha[i], ha2[i]) { + t.Fatalf( + "failed to unmarshal at element %d; got %x, expected %x", + i, ha[i], ha2[i], + ) + } + } +} diff --git a/pkg/encoders/text/wrap.go b/pkg/encoders/text/wrap.go new file mode 100644 index 0000000..f09b948 --- /dev/null +++ b/pkg/encoders/text/wrap.go @@ -0,0 +1,88 @@ +package text + +// AppendBytesClosure is a function type for appending data from a source to a destination and +// returning the appended-to slice. +type AppendBytesClosure func(dst, src []byte) []byte + +// AppendClosure is a simple append where the caller appends to the destination and returns the +// appended-to slice. +type AppendClosure func(dst []byte) []byte + +// Unquote removes the quotes around a slice of bytes. +func Unquote(b []byte) []byte { return b[1 : len(b)-1] } + +// Noop simply appends the source to the destination slice and returns it. +func Noop(dst, src []byte) []byte { return append(dst, src...) } + +// AppendQuote appends a source of bytes, that have been processed by an AppendBytesClosure and +// returns the appended-to slice. +func AppendQuote(dst, src []byte, ac AppendBytesClosure) []byte { + dst = append(dst, '"') + dst = ac(dst, src) + dst = append(dst, '"') + return dst +} + +// Quote simply quotes a provided source and attaches it to the provided destination slice. +func Quote(dst, src []byte) []byte { return AppendQuote(dst, src, Noop) } + +// AppendSingleQuote appends a provided AppendBytesClosure's output from a given source of +// bytes, wrapped in single quotes ”. +func AppendSingleQuote(dst, src []byte, ac AppendBytesClosure) []byte { + dst = append(dst, '\'') + dst = ac(dst, src) + dst = append(dst, '\'') + return dst +} + +// AppendBackticks appends a provided AppendBytesClosure's output from a given source of +// bytes, wrapped in backticks “. +func AppendBackticks(dst, src []byte, ac AppendBytesClosure) []byte { + dst = append(dst, '`') + dst = ac(dst, src) + dst = append(dst, '`') + return dst +} + +// AppendBrace appends a provided AppendBytesClosure's output from a given source of +// bytes, wrapped in braces (). +func AppendBrace(dst, src []byte, ac AppendBytesClosure) []byte { + dst = append(dst, '(') + dst = ac(dst, src) + dst = append(dst, ')') + return dst +} + +// AppendParenthesis appends a provided AppendBytesClosure's output from a given source of +// bytes, wrapped in parentheses {}. +func AppendParenthesis(dst, src []byte, ac AppendBytesClosure) []byte { + dst = append(dst, '{') + dst = ac(dst, src) + dst = append(dst, '}') + return dst +} + +// AppendBracket appends a provided AppendBytesClosure's output from a given source of +// bytes, wrapped in brackets []. +func AppendBracket(dst, src []byte, ac AppendBytesClosure) []byte { + dst = append(dst, '[') + dst = ac(dst, src) + dst = append(dst, ']') + return dst +} + +// AppendList appends an input source bytes processed by an AppendBytesClosure and separates +// elements with the given separator byte. +func AppendList( + dst []byte, src [][]byte, separator byte, + ac AppendBytesClosure, +) []byte { + last := len(src) - 1 + for i := range src { + dst = append(dst, ac(dst, src[i])...) + if i < last { + dst = append(dst, separator) + } + } + return dst +} diff --git a/pkg/utils/bufpool/bufpool.go b/pkg/utils/bufpool/bufpool.go new file mode 100644 index 0000000..83e376c --- /dev/null +++ b/pkg/utils/bufpool/bufpool.go @@ -0,0 +1,78 @@ +package bufpool + +import ( + "fmt" + "sync" + "unsafe" + + "lol.mleku.dev/log" + "next.orly.dev/pkg/utils/units" +) + +const ( + // BufferSize is the size of each buffer in the pool (1kb) + BufferSize = units.Kb / 2 +) + +type B []byte + +func (b B) ToBytes() []byte { return b } + +var Pool = sync.Pool{ + New: func() interface{} { + // Create a new buffer when the pool is empty + b := make([]byte, 0, BufferSize) + log.T.C( + func() string { + ptr := unsafe.SliceData(b) + return fmt.Sprintf("creating buffer at: %p", ptr) + }, + ) + return B(b) + }, +} + +// Get returns a buffer from the pool or creates a new one if the pool is empty. +// +// Example usage: +// +// buf := bufpool.Get() +// defer bufpool.Put(buf) +// // Use buf... +func Get() B { + b := Pool.Get().(B) + log.T.C( + func() string { + ptr := unsafe.SliceData(b) + return fmt.Sprintf("getting buffer at: %p", ptr) + }, + ) + return b +} + +// Put returns a buffer to the pool. +// Buffers should be returned to the pool when no longer needed to allow reuse. +func Put(b B) { + for i := range b { + (b)[i] = 0 + } + b = b[:0] + log.T.C( + func() string { + ptr := unsafe.SliceData(b) + return fmt.Sprintf("returning to buffer: %p", ptr) + }, + ) + Pool.Put(b) +} + +// PutBytes returns a buffer was not necessarily created by Get(). +func PutBytes(b []byte) { + log.T.C( + func() string { + ptr := unsafe.SliceData(b) + return fmt.Sprintf("returning bytes to buffer: %p", ptr) + }, + ) + Put(b) +} diff --git a/pkg/utils/bufpool/bufpool_test.go b/pkg/utils/bufpool/bufpool_test.go new file mode 100644 index 0000000..0bfec55 --- /dev/null +++ b/pkg/utils/bufpool/bufpool_test.go @@ -0,0 +1,71 @@ +package bufpool + +import ( + "testing" +) + +func TestBufferPoolGetPut(t *testing.T) { + // Get a buffer from the pool + buf1 := Get() + + // Verify the buffer is the correct size + if len(*buf1) != BufferSize { + t.Errorf("Expected buffer size of %d, got %d", BufferSize, len(*buf1)) + } + + // Write some data to the buffer + (*buf1)[0] = 42 + + // Return the buffer to the pool + Put(buf1) + + // Get another buffer, which should be the same one we just returned + buf2 := Get() + + // Buffer may or may not be cleared, but we should be able to use it + // Let's check if we have the expected buffer size + if len(*buf2) != BufferSize { + t.Errorf("Expected buffer size of %d, got %d", BufferSize, len(*buf2)) + } +} + +func TestMultipleBuffers(t *testing.T) { + // Get multiple buffers at once to ensure the pool can handle it + const numBuffers = 10 + buffers := make([]B, numBuffers) + + // Get buffers from the pool + for i := 0; i < numBuffers; i++ { + buffers[i] = Get() + // Verify each buffer is the correct size + if len(*buffers[i]) != BufferSize { + t.Errorf( + "Buffer %d: Expected size of %d, got %d", i, BufferSize, + len(*buffers[i]), + ) + } + } + + // Return all buffers to the pool + for i := 0; i < numBuffers; i++ { + Put(buffers[i]) + } +} + +func BenchmarkGetPut(b *testing.B) { + for i := 0; i < b.N; i++ { + buf := Get() + Put(buf) + } +} + +func BenchmarkGetPutParallel(b *testing.B) { + b.RunParallel( + func(pb *testing.PB) { + for pb.Next() { + buf := Get() + Put(buf) + } + }, + ) +}