Add core packages, configuration system, and initial application structure

This commit is contained in:
2025-08-21 11:04:03 +01:00
parent b8db587d7b
commit ecaf52b98f
18 changed files with 12889 additions and 0 deletions

View File

@@ -0,0 +1,47 @@
package event
// E is the primary datatype of nostr. This is the form of the structure that
// defines its JSON string-based format.
type E struct {
// ID is the SHA256 hash of the canonical encoding of the event in binary format
ID []byte
// Pubkey is the public key of the event creator in binary format
Pubkey []byte
// CreatedAt is the UNIX timestamp of the event according to the event
// creator (never trust a timestamp!)
CreatedAt int64
// Kind is the nostr protocol code for the type of event. See kind.T
Kind uint16
// Tags are a list of tags, which are a list of strings usually structured
// as a 3-layer scheme indicating specific features of an event.
Tags [][]byte
// Content is an arbitrary string that can contain anything, but usually
// conforming to a specification relating to the Kind and the Tags.
Content []byte
// Sig is the signature on the ID hash that validates as coming from the
// Pubkey in binary format.
Sig []byte
}
// S is an array of event.E that sorts in reverse chronological order.
type S []*E
// Len returns the length of the event.Es.
func (ev S) Len() int { return len(ev) }
// Less returns whether the first is newer than the second (larger unix
// timestamp).
func (ev S) Less(i, j int) bool { return ev[i].CreatedAt > ev[j].CreatedAt }
// Swap two indexes of the event.Es with each other.
func (ev S) Swap(i, j int) { ev[i], ev[j] = ev[j], ev[i] }
// C is a channel that carries event.E.
type C chan *E

View File

@@ -0,0 +1,10 @@
// Package examples is an embedded jsonl format of a collection of events
// intended to be used to test an event codec.
package examples
import (
_ "embed"
)
//go:embed out.jsonl
var Cache []byte

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,20 @@
// Package main is a generator for the base10000 (4 digit) encoding of the ints
// library.
package main
import (
"fmt"
"os"
"lol.mleku.dev/chk"
)
func main() {
fh, err := os.Create("pkg/ints/base10k.txt")
if chk.E(err) {
panic(err)
}
for i := range 10000 {
fmt.Fprintf(fh, "%04d", i)
}
}

135
pkg/encoders/ints/ints.go Normal file
View File

@@ -0,0 +1,135 @@
// Package ints is an optimised encoder for decimal numbers in ASCII format,
// that simplifies and accelerates encoding and decoding decimal strings. It is
// faster than strconv in part because it uses a base of 10000 and a lookup
// table.
package ints
import (
_ "embed"
"io"
"golang.org/x/exp/constraints"
"lol.mleku.dev/errorf"
)
// run this to regenerate (pointlessly) the base 10 array of 4 places per entry
//go:generate go run ./gen/.
//go:embed base10k.txt
var base10k []byte
const base = 10000
// T is an integer with a fast codec to decimal ASCII.
type T struct {
N uint64
}
func New[V constraints.Integer](n V) *T {
return &T{uint64(n)}
}
// Uint64 returns the int.T as a uint64 (the base type)
func (n *T) Uint64() uint64 { return n.N }
// Int64 returns an int64 from the base number (may cause truncation)
func (n *T) Int64() int64 { return int64(n.N) }
// Uint16 returns an uint16 from the base number (may cause truncation)
func (n *T) Uint16() uint16 { return uint16(n.N) }
var powers = []*T{
{1},
{1_0000},
{1_0000_0000},
{1_0000_0000_0000},
{1_0000_0000_0000_0000},
}
const zero = '0'
const nine = '9'
// Marshal the int.T into a byte string.
func (n *T) Marshal(dst []byte) (b []byte) {
nn := n.N
b = dst
if n.N == 0 {
b = append(b, '0')
return
}
var i int
var trimmed bool
k := len(powers)
for k > 0 {
k--
q := n.N / powers[k].N
if !trimmed && q == 0 {
continue
}
offset := q * 4
bb := base10k[offset : offset+4]
if !trimmed {
for i = range bb {
if bb[i] != '0' {
bb = bb[i:]
trimmed = true
break
}
}
}
b = append(b, bb...)
n.N = n.N - q*powers[k].N
}
n.N = nn
return
}
// Unmarshal reads a string, which must be a positive integer no larger than math.MaxUint64,
// skipping any non-numeric content before it.
//
// Note that leading zeros are not considered valid, but basically no such thing as machine
// generated JSON integers with leading zeroes. Until this is disproven, this is the fastest way
// to read a positive json integer, and a leading zero is decoded as a zero, and the remainder
// returned.
func (n *T) Unmarshal(b []byte) (r []byte, err error) {
if len(b) < 1 {
err = errorf.E("zero length number")
return
}
var sLen int
if b[0] == zero {
r = b[1:]
n.N = 0
return
}
// skip non-number characters
for i, v := range b {
if v >= '0' && v <= '9' {
b = b[i:]
break
}
}
if len(b) == 0 {
err = io.EOF
return
}
// count the digits
for ; sLen < len(b) && b[sLen] >= zero && b[sLen] <= nine && b[sLen] != ','; sLen++ {
}
if sLen == 0 {
err = errorf.E("zero length number")
return
}
if sLen > 20 {
err = errorf.E("too big number for uint64")
return
}
// the length of the string found
r = b[sLen:]
b = b[:sLen]
for _, ch := range b {
ch -= zero
n.N = n.N*10 + uint64(ch)
}
return
}

View File

@@ -0,0 +1,87 @@
package ints
import (
"math"
"strconv"
"testing"
"lol.mleku.dev/chk"
"lukechampine.com/frand"
)
func TestMarshalUnmarshal(t *testing.T) {
b := make([]byte, 0, 8)
var rem []byte
var n *T
var err error
for _ = range 10000000 {
n = New(uint64(frand.Intn(math.MaxInt64)))
b = n.Marshal(b)
m := New(0)
if rem, err = m.Unmarshal(b); chk.E(err) {
t.Fatal(err)
}
if n.N != m.N {
t.Fatalf("failed to convert to int64 at %d %s %d", n.N, b, m.N)
}
if len(rem) > 0 {
t.Fatalf("leftover bytes after converting back: '%s'", rem)
}
b = b[:0]
}
}
func BenchmarkByteStringToInt64(bb *testing.B) {
b := make([]byte, 0, 19)
var i int
const nTests = 10000000
testInts := make([]*T, nTests)
for i = range nTests {
testInts[i] = New(frand.Intn(math.MaxInt64))
}
bb.Run(
"Marshal", func(bb *testing.B) {
bb.ReportAllocs()
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
b = n.Marshal(b)
b = b[:0]
}
},
)
bb.Run(
"Itoa", func(bb *testing.B) {
bb.ReportAllocs()
var s string
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
s = strconv.Itoa(int(n.N))
_ = s
}
},
)
bb.Run(
"MarshalUnmarshal", func(bb *testing.B) {
bb.ReportAllocs()
m := New(0)
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
b = m.Marshal(b)
_, _ = n.Unmarshal(b)
b = b[:0]
}
},
)
bb.Run(
"ItoaAtoi", func(bb *testing.B) {
bb.ReportAllocs()
var s string
for i = 0; i < bb.N; i++ {
n := testInts[i%10000]
s = strconv.Itoa(int(n.N))
_, _ = strconv.Atoi(s)
}
},
)
}

127
pkg/encoders/text/escape.go Normal file
View File

@@ -0,0 +1,127 @@
package text
// NostrEscape for JSON encoding according to RFC8259.
//
// This is the efficient implementation based on the NIP-01 specification:
//
// To prevent implementation differences from creating a different event ID for
// the same event, the following rules MUST be followed while serializing:
//
// No whitespace, line breaks or other unnecessary formatting should be included
// in the output JSON. No characters except the following should be escaped, and
// instead should be included verbatim:
//
// - A line break, 0x0A, as \n
// - A double quote, 0x22, as \"
// - A backslash, 0x5C, as \\
// - A carriage return, 0x0D, as \r
// - A tab character, 0x09, as \t
// - A backspace, 0x08, as \b
// - A form feed, 0x0C, as \f
//
// UTF-8 should be used for encoding.
func NostrEscape(dst, src []byte) []byte {
l := len(src)
for i := 0; i < l; i++ {
c := src[i]
switch {
case c == '"':
dst = append(dst, '\\', '"')
case c == '\\':
// if i+1 < l && src[i+1] == 'u' || i+1 < l && src[i+1] == '/' {
if i+1 < l && src[i+1] == 'u' {
dst = append(dst, '\\')
} else {
dst = append(dst, '\\', '\\')
}
case c == '\b':
dst = append(dst, '\\', 'b')
case c == '\t':
dst = append(dst, '\\', 't')
case c == '\n':
dst = append(dst, '\\', 'n')
case c == '\f':
dst = append(dst, '\\', 'f')
case c == '\r':
dst = append(dst, '\\', 'r')
default:
dst = append(dst, c)
}
}
return dst
}
// NostrUnescape reverses the operation of NostrEscape except instead of
// appending it to the provided slice, it rewrites it, eliminating a memory
// copy. Keep in mind that the original JSON will be mangled by this operation,
// but the resultant slices will cost zero allocations.
func NostrUnescape(dst []byte) (b []byte) {
var r, w int
for ; r < len(dst); r++ {
if dst[r] == '\\' {
r++
c := dst[r]
switch {
// nip-01 specifies the following single letter C-style escapes for control
// codes under 0x20.
//
// no others are specified but must be preserved, so only these can be
// safely decoded at runtime as they must be re-encoded when marshalled.
case c == '"':
dst[w] = '"'
w++
case c == '\\':
dst[w] = '\\'
w++
case c == 'b':
dst[w] = '\b'
w++
case c == 't':
dst[w] = '\t'
w++
case c == 'n':
dst[w] = '\n'
w++
case c == 'f':
dst[w] = '\f'
w++
case c == 'r':
dst[w] = '\r'
w++
// special cases for non-nip-01 specified json escapes (must be preserved for ID
// generation).
case c == 'u':
dst[w] = '\\'
w++
dst[w] = 'u'
w++
case c == '/':
dst[w] = '\\'
w++
dst[w] = '/'
w++
// special case for octal escapes (must be preserved for ID generation).
case c >= '0' && c <= '9':
dst[w] = '\\'
w++
dst[w] = c
w++
// anything else after a reverse solidus just preserve it.
default:
dst[w] = dst[r]
w++
dst[w] = c
w++
}
} else {
dst[w] = dst[r]
w++
}
}
b = dst[:w]
return
}

View File

@@ -0,0 +1,459 @@
package text
import (
"testing"
"lol.mleku.dev/chk"
"next.orly.dev/pkg/crypto/sha256"
"lukechampine.com/frand"
)
func TestUnescapeByteString(t *testing.T) {
b := make([]byte, 256)
for i := range b {
b[i] = byte(i)
}
escaped := NostrEscape(nil, b)
unescaped := NostrUnescape(escaped)
if string(b) != string(unescaped) {
t.Log(b)
t.Log(unescaped)
t.FailNow()
}
}
func GenRandString(l int, src *frand.RNG) (str []byte) {
return src.Bytes(l)
}
var seed = sha256.Sum256(
[]byte(`
The tao that can be told
is not the eternal Tao
The name that can be named
is not the eternal Name
The unnamable is the eternally real
Naming is the origin of all particular things
Free from desire, you realize the mystery
Caught in desire, you see only the manifestations
Yet mystery and manifestations arise from the same source
This source is called darkness
Darkness within darkness
The gateway to all understanding
`),
)
var src = frand.NewCustom(seed[:], 32, 12)
func TestRandomEscapeByteString(t *testing.T) {
// this is a kind of fuzz test, does a massive number of iterations of
// random content that ensures the escaping is correct without creating a
// fixed set of test vectors.
for i := 0; i < 1000; i++ {
l := src.Intn(1<<8) + 32
s1 := GenRandString(l, src)
s2 := make([]byte, l)
orig := make([]byte, l)
copy(s2, s1)
copy(orig, s1)
// first we are checking our implementation comports to the one from go-nostr.
escapeStringVersion := NostrEscape([]byte{}, s1)
escapeJSONStringAndWrapVersion := NostrEscape(nil, s2)
if len(escapeJSONStringAndWrapVersion) != len(escapeStringVersion) {
t.Logf(
"escapeString\nlength: %d\n%s\n%v\n",
len(escapeStringVersion), string(escapeStringVersion),
escapeStringVersion,
)
t.Logf(
"escapJSONStringAndWrap\nlength: %d\n%s\n%v\n",
len(escapeJSONStringAndWrapVersion),
escapeJSONStringAndWrapVersion,
escapeJSONStringAndWrapVersion,
)
t.FailNow()
}
for i := range escapeStringVersion {
if i > len(escapeJSONStringAndWrapVersion) {
t.Fatal("escapeString version is shorter")
}
if escapeStringVersion[i] != escapeJSONStringAndWrapVersion[i] {
t.Logf(
"escapeString version differs at index %d from "+
"escapeJSONStringAndWrap version\n%s\n%s\n%v\n%v", i,
escapeStringVersion[i-4:],
escapeJSONStringAndWrapVersion[i-4:],
escapeStringVersion[i-4:],
escapeJSONStringAndWrapVersion[i-4:],
)
t.Logf(
"escapeString\nlength: %d %s\n",
len(escapeStringVersion), escapeStringVersion,
)
t.Logf(
"escapJSONStringAndWrap\nlength: %d %s\n",
len(escapeJSONStringAndWrapVersion),
escapeJSONStringAndWrapVersion,
)
t.Logf(
"got '%s' %d expected '%s' %d\n",
string(escapeJSONStringAndWrapVersion[i]),
escapeJSONStringAndWrapVersion[i],
string(escapeStringVersion[i]),
escapeStringVersion[i],
)
t.FailNow()
}
}
// next, unescape the output and see if it matches the original
unescaped := NostrUnescape(escapeJSONStringAndWrapVersion)
// t.Logf("unescaped: \n%s\noriginal: \n%s", unescaped, orig)
if string(unescaped) != string(orig) {
t.Fatalf(
"\ngot %d %v\nexpected %d %v\n",
len(unescaped),
unescaped,
len(orig),
orig,
)
}
}
}
func BenchmarkNostrEscapeNostrUnescape(b *testing.B) {
const size = 65536
b.Run(
"frand64k", func(b *testing.B) {
b.ReportAllocs()
in := make([]byte, size)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
}
},
)
b.Run(
"NostrEscape64k", func(b *testing.B) {
b.ReportAllocs()
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
out = out[:0]
}
},
)
b.Run(
"NostrEscapeNostrUnescape64k", func(b *testing.B) {
b.ReportAllocs()
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
in = in[:0]
out = NostrUnescape(out)
out = out[:0]
}
},
)
b.Run(
"frand32k", func(b *testing.B) {
b.ReportAllocs()
size := size / 2
in := make([]byte, size)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
}
},
)
b.Run(
"NostrEscape32k", func(b *testing.B) {
b.ReportAllocs()
size := size / 2
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
out = out[:0]
}
},
)
b.Run(
"NostrEscapeNostrUnescape32k", func(b *testing.B) {
b.ReportAllocs()
size := size / 2
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
in = in[:0]
out = NostrUnescape(out)
out = out[:0]
}
},
)
b.Run(
"frand16k", func(b *testing.B) {
b.ReportAllocs()
size := size / 4
in := make([]byte, size)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
}
},
)
b.Run(
"NostrEscape16k", func(b *testing.B) {
b.ReportAllocs()
size := size / 4
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
out = out[:0]
}
},
)
b.Run(
"NostrEscapeNostrUnescape16k", func(b *testing.B) {
b.ReportAllocs()
size := size / 4
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
in = in[:0]
out = NostrUnescape(out)
out = out[:0]
}
},
)
b.Run(
"frand8k", func(b *testing.B) {
b.ReportAllocs()
size := size / 8
in := make([]byte, size)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
}
},
)
b.Run(
"NostrEscape8k", func(b *testing.B) {
b.ReportAllocs()
size := size / 8
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
out = out[:0]
}
},
)
b.Run(
"NostrEscapeNostrUnescape8k", func(b *testing.B) {
b.ReportAllocs()
size := size / 8
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
in = in[:0]
out = NostrUnescape(out)
out = out[:0]
}
},
)
b.Run(
"frand4k", func(b *testing.B) {
b.ReportAllocs()
size := size / 16
in := make([]byte, size)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
}
},
)
b.Run(
"NostrEscape4k", func(b *testing.B) {
b.ReportAllocs()
size := size / 16
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
out = out[:0]
}
},
)
b.Run(
"NostrEscapeNostrUnescape4k", func(b *testing.B) {
b.ReportAllocs()
size := size / 16
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
in = in[:0]
out = NostrUnescape(out)
out = out[:0]
}
},
)
b.Run(
"frand2k", func(b *testing.B) {
b.ReportAllocs()
size := size / 32
in := make([]byte, size)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
}
},
)
b.Run(
"NostrEscape2k", func(b *testing.B) {
b.ReportAllocs()
size := size / 32
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
out = out[:0]
}
},
)
b.Run(
"NostrEscapeNostrUnescape2k", func(b *testing.B) {
b.ReportAllocs()
size := size / 32
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
in = in[:0]
out = NostrUnescape(out)
out = out[:0]
}
},
)
b.Run(
"frand1k", func(b *testing.B) {
b.ReportAllocs()
size := size / 64
in := make([]byte, size)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
}
},
)
b.Run(
"NostrEscape1k", func(b *testing.B) {
b.ReportAllocs()
size := size / 64
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
out = out[:0]
}
},
)
b.Run(
"NostrEscapeNostrUnescape1k", func(b *testing.B) {
b.ReportAllocs()
size := size / 64
in := make([]byte, size)
out := make([]byte, size*2)
var err error
for i := 0; i < b.N; i++ {
if _, err = frand.Read(in); chk.E(err) {
b.Fatal(err)
}
out = NostrEscape(out, in)
in = in[:0]
out = NostrUnescape(out)
out = out[:0]
}
},
)
}