mleku/next.orly.dev
1
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove bufpool references and unused imports, optimize memory operations.

Browse Source 
- Removed `bufpool` usage throughout `tag`, `tags`, and `event` packages for memory efficiency.
- Replaced in-place buffer modifications with independent, deep-copied allocations to prevent unintended mutations.
- Added new `Clone` method for deep copying `event.E`.
- Ensured valid JSON emission for nil `Tags` in `event` marshaling.
- Introduced `cmd/stresstest` for relay stress-testing with detailed workload generation and query simulation.
This commit is contained in:
mleku
2025-09-19 16:17:44 +01:00
parent 49a172820a
commit 22cde96f3f
7 changed files with 681 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/encoders/envelopes/eventenvelope/eventenvelope.go
View File

@@ -7,12 +7,10 @@ import (
"lol.mleku.dev/chk"
"lol.mleku.dev/errorf"
"lol.mleku.dev/log"
"next.orly.dev/pkg/encoders/envelopes"
"next.orly.dev/pkg/encoders/event"
"next.orly.dev/pkg/encoders/text"
"next.orly.dev/pkg/interfaces/codec"
"next.orly.dev/pkg/utils/bufpool"
"next.orly.dev/pkg/utils/constraints"
"next.orly.dev/pkg/utils/units"
)
@@ -76,8 +74,8 @@ func (en *Submission) Unmarshal(b []byte) (r []byte, err error) {
if r, err = en.E.Unmarshal(r); chk.T(err) {
return
}
buf := bufpool.Get()
r = en.E.Marshal(buf)
// after parsing the event object, r points just after the event JSON
// now skip to the end of the envelope (consume comma/closing bracket etc.)
if r, err = envelopes.SkipToTheEnd(r); chk.E(err) {
return
}
@@ -162,7 +160,6 @@ func (en *Result) Unmarshal(b []byte) (r []byte, err error) {
return
}
en.Event = event.New()
log.I.F("unmarshal: '%s'", b)
if r, err = en.Event.Unmarshal(r); err != nil {
return
}

88
pkg/encoders/event/event.go
View File

@@ -15,13 +15,10 @@ import (
"next.orly.dev/pkg/encoders/tag"
"next.orly.dev/pkg/encoders/text"
"next.orly.dev/pkg/utils"
"next.orly.dev/pkg/utils/bufpool"
)
// E is the primary datatype of nostr. This is the form of the structure that
// defines its JSON string-based format. Always use New() and Free() to create
// and free event.E to take advantage of the bufpool which greatly improves
// memory allocation behaviour when encoding and decoding nostr events.
// defines its JSON string-based format.
//
// WARNING: DO NOT use json.Marshal with this type because it will not properly
// encode <, >, and & characters due to legacy bullcrap in the encoding/json
@@ -57,10 +54,6 @@ type E struct {
// Sig is the signature on the ID hash that validates as coming from the
// Pubkey in binary format.
Sig []byte
// b is the decode buffer for the event.E. this is where the UnmarshalJSON
// will source the memory to store all of the fields except for the tags.
b bufpool.B
}
var (
@@ -73,25 +66,66 @@ var (
jSig = []byte("sig")
)
// New returns a new event.E. The returned event.E should be freed with Free()
// to return the unmarshalling buffer to the bufpool.
// New returns a new event.E.
func New() *E {
return &E{
b: bufpool.Get(),
}
return &E{}
}
// Free returns the event.E to the pool, as well as nilling all of the fields.
// This should hint to the GC that the event.E can be freed, and the memory
// reused. The decode buffer will be returned to the pool for reuse.
// Free nils all of the fields to hint to the GC that the event.E can be freed.
func (ev *E) Free() {
bufpool.Put(ev.b)
ev.ID = nil
ev.Pubkey = nil
ev.Tags = nil
ev.Content = nil
ev.Sig = nil
ev.b = nil
}
// Clone creates a deep copy of the event with independent memory allocations.
// The clone does not use bufpool, ensuring it has a separate lifetime from
// the original event. This prevents corruption when the original is freed
// while the clone is still in use (e.g., in asynchronous delivery).
func (ev *E) Clone() *E {
clone := &E{
CreatedAt: ev.CreatedAt,
Kind: ev.Kind,
}
// Deep copy all byte slices with independent memory
if ev.ID != nil {
clone.ID = make([]byte, len(ev.ID))
copy(clone.ID, ev.ID)
}
if ev.Pubkey != nil {
clone.Pubkey = make([]byte, len(ev.Pubkey))
copy(clone.Pubkey, ev.Pubkey)
}
if ev.Content != nil {
clone.Content = make([]byte, len(ev.Content))
copy(clone.Content, ev.Content)
}
if ev.Sig != nil {
clone.Sig = make([]byte, len(ev.Sig))
copy(clone.Sig, ev.Sig)
}
// Deep copy tags
if ev.Tags != nil {
clone.Tags = tag.NewS()
for _, tg := range *ev.Tags {
if tg != nil {
// Create new tag with deep-copied elements
newTag := tag.NewWithCap(len(tg.T))
for _, element := range tg.T {
newElement := make([]byte, len(element))
copy(newElement, element)
newTag.T = append(newTag.T, newElement)
}
clone.Tags.Append(newTag)
}
}
}
return clone
}
// EstimateSize returns a size for the event that allows for worst case scenario
@@ -135,6 +169,9 @@ func (ev *E) Marshal(dst []byte) (b []byte) {
b = append(b, `":`...)
if ev.Tags != nil {
b = ev.Tags.Marshal(b)
} else {
// Emit empty array for nil tags to keep JSON valid
b = append(b, '[', ']')
}
b = append(b, `,"`...)
b = append(b, jContent...)
@@ -151,29 +188,22 @@ func (ev *E) Marshal(dst []byte) (b []byte) {
// MarshalJSON marshals an event.E into a JSON byte string.
//
// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use.
//
// WARNING: if json.Marshal is called in the hopes of invoking this function on
// an event, if it has <, > or * in the content or tags they are escaped into
// unicode escapes and break the event ID. Call this function directly in order
// to bypass this issue.
func (ev *E) MarshalJSON() (b []byte, err error) {
b = bufpool.Get()
b = ev.Marshal(b[:0])
b = ev.Marshal(nil)
return
}
func (ev *E) Serialize() (b []byte) {
b = bufpool.Get()
b = ev.Marshal(b[:0])
b = ev.Marshal(nil)
return
}
// Unmarshal unmarshalls a JSON string into an event.E.
//
// Call ev.Free() to return the provided buffer to the bufpool afterwards.
func (ev *E) Unmarshal(b []byte) (rem []byte, err error) {
log.I.F("Unmarshal\n%s\n", string(b))
key := make([]byte, 0, 9)
for ; len(b) > 0; b = b[1:] {
// Skip whitespace
@@ -185,7 +215,6 @@ func (ev *E) Unmarshal(b []byte) (rem []byte, err error) {
goto BetweenKeys
}
}
log.I.F("start")
goto eof
BetweenKeys:
for ; len(b) > 0; b = b[1:] {
@@ -198,7 +227,6 @@ BetweenKeys:
goto InKey
}
}
log.I.F("BetweenKeys")
goto eof
InKey:
for ; len(b) > 0; b = b[1:] {
@@ -208,7 +236,6 @@ InKey:
}
key = append(key, b[0])
}
log.I.F("InKey")
goto eof
InKV:
for ; len(b) > 0; b = b[1:] {
@@ -221,7 +248,6 @@ InKV:
goto InVal
}
}
log.I.F("InKV")
goto eof
InVal:
// Skip whitespace before value

25
pkg/encoders/tag/tag.go
View File

@@ -9,7 +9,6 @@ import (
"lol.mleku.dev/errorf"
"next.orly.dev/pkg/encoders/text"
utils "next.orly.dev/pkg/utils"
"next.orly.dev/pkg/utils/bufpool"
)
// The tag position meanings, so they are clear when reading.
@@ -21,18 +20,17 @@ const (
type T struct {
T [][]byte
b bufpool.B
}
func New() *T { return &T{b: bufpool.Get()} }
func New() *T { return &T{} }
func NewFromBytesSlice(t ...[]byte) (tt *T) {
tt = &T{T: t, b: bufpool.Get()}
tt = &T{T: t}
return
}
func NewFromAny(t ...any) (tt *T) {
tt = &T{b: bufpool.Get()}
tt = &T{}
for _, v := range t {
switch vv := v.(type) {
case []byte:
@@ -47,11 +45,10 @@ func NewFromAny(t ...any) (tt *T) {
}
func NewWithCap(c int) *T {
return &T{T: make([][]byte, 0, c), b: bufpool.Get()}
return &T{T: make([][]byte, 0, c)}
}
func (t *T) Free() {
bufpool.Put(t.b)
t.T = nil
}
@@ -99,18 +96,12 @@ func (t *T) Marshal(dst []byte) (b []byte) {
// in an event as you will have a bad time. Use the json.Marshal function in the
// pkg/encoders/json package instead, this has a fork of the json library that
// disables html escaping for json.Marshal.
//
// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use.
func (t *T) MarshalJSON() (b []byte, err error) {
b = bufpool.Get()
b = t.Marshal(b)
b = t.Marshal(nil)
return
}
// Unmarshal decodes a standard minified JSON array of strings to a tags.T.
//
// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use if it
// was originally created using bufpool.Get().
func (t *T) Unmarshal(b []byte) (r []byte, err error) {
var inQuotes, openedBracket bool
var quoteStart int
@@ -127,7 +118,11 @@ func (t *T) Unmarshal(b []byte) (r []byte, err error) {
i++
} else if b[i] == '"' {
inQuotes = false
t.T = append(t.T, text.NostrUnescape(b[quoteStart:i]))
// Copy the quoted substring before unescaping so we don't mutate the
// original JSON buffer in-place (which would corrupt subsequent parsing).
copyBuf := make([]byte, i-quoteStart)
copy(copyBuf, b[quoteStart:i])
t.T = append(t.T, text.NostrUnescape(copyBuf))
}
}
if !openedBracket || inQuotes {

6
pkg/encoders/tag/tags.go
View File

@@ -5,7 +5,6 @@ import (
"lol.mleku.dev/chk"
"next.orly.dev/pkg/utils"
"next.orly.dev/pkg/utils/bufpool"
)
// S is a list of tag.T - which are lists of string elements with ordering and
@@ -70,10 +69,7 @@ func (s *S) ContainsAny(tagName []byte, values [][]byte) bool {
}
// MarshalJSON encodes a tags.T appended to a provided byte slice in JSON form.
//
// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use.
func (s *S) MarshalJSON() (b []byte, err error) {
b = bufpool.Get()
b = append(b, '[')
for i, ss := range *s {
b = ss.Marshal(b)
@@ -100,8 +96,6 @@ func (s *S) Marshal(dst []byte) (b []byte) {
// UnmarshalJSON a tags.T from a provided byte slice and return what remains
// after the end of the array.
//
// Call bufpool.PutBytes(b) to return the buffer to the bufpool after use.
func (s *S) UnmarshalJSON(b []byte) (err error) {
_, err = s.Unmarshal(b)
return

5
pkg/encoders/text/helpers.go
View File

@@ -94,7 +94,10 @@ func UnmarshalQuoted(b []byte) (content, rem []byte, err error) {
if !escaping {
rem = rem[1:]
content = content[:contentLen]
content = NostrUnescape(content)
// Create a copy of the content to avoid corrupting the original input buffer
contentCopy := make([]byte, len(content))
copy(contentCopy, content)
content = NostrUnescape(contentCopy)
return
}
contentLen++