define key codec and key types

codec/codec.go

@@ -0,0 +1,10 @@
+package codec
+
+import (
+	"io"
+)
+
+type I interface {
+	MarshalWrite(w io.Writer) (err error)
+	UnmarshalRead(r io.Reader) (err error)
+}

database/keys/keys.go

@@ -0,0 +1,165 @@
+package keys
+
+import (
+	"io"
+	"reflect"
+
+	"github.com/mleku/manifold/chk"
+	"github.com/mleku/manifold/codec"
+	"github.com/mleku/manifold/database/keys/types/fullid"
+	"github.com/mleku/manifold/database/keys/types/fulltext"
+	"github.com/mleku/manifold/database/keys/types/identhash"
+	"github.com/mleku/manifold/database/keys/types/idhash"
+	. "github.com/mleku/manifold/database/keys/types/number"
+	"github.com/mleku/manifold/database/keys/types/prefix"
+	"github.com/mleku/manifold/database/keys/types/pubhash"
+	"github.com/mleku/manifold/database/prefixes"
+)
+
+type Encs []codec.I
+
+// T is a wrapper around an array of codec.I. The caller provides the Encs so
+// they can then call the accessor function of the codec.I implementation.
+type T struct {
+	Encs
+}
+
+// New creates a new indexes. The helper functions below have an encode and
+// decode variant, the decode variant does not add the prefix encoder because it
+// has been read by prefixes.Identify.
+func New(encoders ...codec.I) (i *T) { return &T{encoders} }
+
+func (t *T) MarshalWrite(w io.Writer) (err error) {
+	for _, e := range t.Encs {
+		if e == nil || reflect.ValueOf(e).IsNil() {
+			// allow a field to be empty, as is needed for search indexes to
+			// create search prefixes.
+			return
+		}
+		if err = e.MarshalWrite(w); chk.E(err) {
+			return
+		}
+	}
+	return
+}
+
+func (t *T) UnmarshalRead(r io.Reader) (err error) {
+	for _, e := range t.Encs {
+		if err = e.UnmarshalRead(r); chk.E(err) {
+			return
+		}
+	}
+	return
+}
+
+func EventVars() (ser *Uint40) {
+	ser = new(Uint40)
+	return
+}
+func EventEnc(ser *Uint40) (enc *T) {
+	return New(prefix.New(prefixes.Event), ser)
+}
+func EventDec(ser *Uint40) (enc *T) {
+	return New(prefix.New(), ser)
+}
+
+func IdVars() (id *idhash.T, ser *Uint40) {
+	id = idhash.New()
+	ser = new(Uint40)
+	return
+}
+func IdEnc(id *idhash.T, ser *Uint40) (enc *T) {
+	return New(prefix.New(prefixes.Id), id, ser)
+}
+func IdSearch(id *idhash.T) (enc *T) {
+	return New(prefix.New(prefixes.Id), id)
+}
+func IdDec(id *idhash.T, ser *Uint40) (enc *T) {
+	return New(prefix.New(), id, ser)
+}
+
+type IdPubkeyTimestamp struct {
+	Ser       *Uint40
+	Id        *fullid.T
+	Pubkey    *pubhash.T
+	Kind      *Uint16
+	Timestamp *Uint64
+}
+
+func IdPubkeyTimestampVars() (ser *Uint40, t *fullid.T, p *pubhash.T, ts *Uint64) {
+	ser = new(Uint40)
+	t = fullid.New()
+	p = pubhash.New()
+	ts = new(Uint64)
+	return
+}
+func IdPubkeyTimestampEnc(ser *Uint40, t *fullid.T, p *pubhash.T, ts *Uint64) (enc *T) {
+	return New(prefix.New(prefixes.IdPubkeyTimestamp), ser, t, p, ts)
+}
+func IdPubkeyTimestampDec(ser *Uint40, t *fullid.T, p *pubhash.T, ts *Uint64) (enc *T) {
+	return New(prefix.New(), ser, t, p, ts)
+}
+
+func TimestampVars() (ts *Uint64, ser *Uint40) {
+	ts = new(Uint64)
+	ser = new(Uint40)
+	return
+}
+func TimestampEnc(ts *Uint64, ser *Uint40) (enc *T) {
+	return New(prefix.New(prefixes.Timestamp), ts, ser)
+}
+func TimestampDec(ts *Uint64, ser *Uint40) (enc *T) {
+	return New(prefix.New(), ts, ser)
+}
+
+func PubkeyTimestampVars() (p *pubhash.T, ts *Uint64, ser *Uint40) {
+	p = pubhash.New()
+	ts = new(Uint64)
+	ser = new(Uint40)
+	return
+}
+func PubkeyTimestampEnc(p *pubhash.T, ts *Uint64, ser *Uint40) (enc *T) {
+	return New(prefix.New(prefixes.PubkeyTimestamp), p, ts, ser)
+}
+func PubkeyTimestampDec(p *pubhash.T, ts *Uint64, ser *Uint40) (enc *T) {
+	return New(prefix.New(), p, ts, ser)
+}
+
+func PubkeyTagTimestampVars() (p *pubhash.T, k, v *identhash.T, ser *Uint40) {
+	k = identhash.New()
+	ser = new(Uint40)
+	return
+}
+func PubkeyTagTimestampEnc(p *pubhash.T, k, v *identhash.T, ser *Uint40) (enc *T) {
+	return New(prefix.New(prefixes.PubkeyTagTimestamp), p, k, v, ser)
+}
+func PubkeyTagTimestampDec(p *pubhash.T, k, v *identhash.T, ser *Uint40) (enc *T) {
+	return New(prefix.New(), p, k, v, ser)
+}
+
+func TagTimestampVars() (k, v *identhash.T, ts *Uint64, ser *Uint40) {
+	k = identhash.New()
+	v = identhash.New()
+	ts = new(Uint64)
+	ser = new(Uint40)
+	return
+}
+func TagTimestampEnc(k, v *identhash.T, ts *Uint64, ser *Uint40) (enc *T) {
+	return New(prefix.New(prefixes.TagTimestamp), k, v, ts, ser)
+}
+func TagTimestampDec(k, v *identhash.T, ts *Uint64, ser *Uint40) (enc *T) {
+	return New(prefix.New(), k, v, ts, ser)
+}
+
+func FullTextWordVars() (fw *fulltext.T, pos *Uint24, ser *Uint40) {
+	fw = fulltext.New()
+	pos = new(Uint24)
+	ser = new(Uint40)
+	return
+}
+func FullTextWordEnc(fw *fulltext.T, pos *Uint24, ser *Uint40) (enc *T) {
+	return New(prefix.New(prefixes.FulltextWord), fw, pos, ser)
+}
+func FullTextWordDec(fw *fulltext.T, pos *Uint24, ser *Uint40) (enc *T) {
+	return New(prefix.New(), fw, pos, ser)
+}

database/keys/types/fullid/fullid.go

@@ -0,0 +1,42 @@
+package fullid
+
+import (
+	"io"
+
+	"github.com/minio/sha256-simd"
+
+	"github.com/mleku/manifold/errorf"
+)
+
+const Len = sha256.Size
+
+type T struct {
+	val []byte
+}
+
+func New() (fi *T) { return &T{make([]byte, Len)} }
+
+func (fi *T) FromId(id []byte) (err error) {
+	if len(id) != Len {
+		err = errorf.E("invalid Id length, got %d require %d", len(id), Len)
+		return
+	}
+	fi.val = id
+	return
+}
+func (fi *T) Bytes() (b []byte) { return fi.val }
+
+func (fi *T) MarshalWrite(w io.Writer) (err error) {
+	_, err = w.Write(fi.val)
+	return
+}
+
+func (fi *T) UnmarshalRead(r io.Reader) (err error) {
+	if len(fi.val) < Len {
+		fi.val = make([]byte, Len)
+	} else {
+		fi.val = fi.val[:Len]
+	}
+	_, err = r.Read(fi.val)
+	return
+}

database/keys/types/fulltext/fulltext.go

@@ -0,0 +1,59 @@
+package fulltext
+
+import (
+	"bytes"
+	"io"
+)
+
+type T struct {
+	val []byte // Contains only the raw word (without the zero-byte marker)
+}
+
+func New() *T {
+	return &T{}
+}
+
+// FromWord stores the word without any modifications
+func (ft *T) FromWord(word []byte) {
+	ft.val = word // Only store the raw word
+}
+
+// Bytes returns the raw word without any end-of-word marker
+func (ft *T) Bytes() []byte {
+	return ft.val
+}
+
+// MarshalWrite writes the word to the writer, appending the zero-byte marker
+func (ft *T) MarshalWrite(w io.Writer) error {
+	// Create a temporary buffer that contains the word and the zero-byte marker
+	temp := append(ft.val, 0x00)
+	_, err := w.Write(temp) // Write the buffer to the writer
+	return err
+}
+
+// UnmarshalRead reads the word from the reader, stopping at the zero-byte marker
+func (ft *T) UnmarshalRead(r io.Reader) error {
+	var buf bytes.Buffer
+	tmp := make([]byte, 1)
+
+	// Read bytes until the zero byte is encountered
+	for {
+		n, err := r.Read(tmp)
+		if n > 0 {
+			if tmp[0] == 0x00 { // Stop on encountering the zero-byte marker
+				break
+			}
+			buf.WriteByte(tmp[0])
+		}
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err // Handle unexpected errors
+		}
+	}
+
+	// Store the raw word without the zero byte
+	ft.val = buf.Bytes()
+	return nil
+}

database/keys/types/fulltext/fulltext_test.go

@@ -0,0 +1,74 @@
+package fulltext_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/mleku/manifold/database/keys/types/fulltext"
+)
+
+func TestT(t *testing.T) {
+	// Test cases: each contains inputs, expected serialized output, and expected result after deserialization.
+	tests := []struct {
+		word            []byte // Input word
+		expectedBytes   []byte // Expected output from Bytes() (raw word)
+		expectedEncoded []byte // Expected serialized (MarshalWrite) output (word + 0x00)
+	}{
+		{[]byte("example"), []byte("example"), []byte("example\x00")},
+		{[]byte("golang"), []byte("golang"), []byte("golang\x00")},
+		{[]byte(""), []byte(""), []byte("\x00")}, // Edge case: empty word
+		{[]byte("123"), []byte("123"), []byte("123\x00")},
+	}
+
+	for _, tt := range tests {
+		// Create a new object and set the word
+		ft := fulltext.New()
+		ft.FromWord(tt.word)
+
+		// Ensure Bytes() returns the correct raw word
+		if got := ft.Bytes(); !bytes.Equal(tt.expectedBytes, got) {
+			t.Errorf("FromWord/Bytes failed: expected %q, got %q", tt.expectedBytes, got)
+		}
+
+		// Test MarshalWrite
+		var buf bytes.Buffer
+		if err := ft.MarshalWrite(&buf); err != nil {
+			t.Fatalf("MarshalWrite failed: %v", err)
+		}
+
+		// Ensure the serialized output matches expectedEncoded
+		if got := buf.Bytes(); !bytes.Equal(tt.expectedEncoded, got) {
+			t.Errorf("MarshalWrite failed: expected %q, got %q", tt.expectedEncoded, got)
+		}
+
+		// Test UnmarshalRead
+		newFt := fulltext.New()
+		if err := newFt.UnmarshalRead(&buf); err != nil {
+			t.Fatalf("UnmarshalRead failed: %v", err)
+		}
+
+		// Ensure the word after decoding matches the original word
+		if got := newFt.Bytes(); !bytes.Equal(tt.expectedBytes, got) {
+			t.Errorf("UnmarshalRead failed: expected %q, got %q", tt.expectedBytes, got)
+		}
+	}
+}
+
+func TestUnmarshalReadHandlesMissingZeroByte(t *testing.T) {
+	// Special case: what happens if the zero-byte marker is missing?
+	data := []byte("incomplete") // No zero-byte at the end
+	reader := bytes.NewReader(data)
+
+	ft := fulltext.New()
+	err := ft.UnmarshalRead(reader)
+
+	// Expect an EOF or similar handling
+	if err == nil {
+		t.Errorf("UnmarshalRead should fail gracefully on missing zero-byte, but it didn't")
+	}
+
+	// Ensure no data is stored in ft.val if no valid end-marker was encountered
+	if got := ft.Bytes(); len(got) != 0 {
+		t.Errorf("UnmarshalRead stored incomplete data: got %q, expected empty", got)
+	}
+}

database/keys/types/identhash/identhash.go

@@ -0,0 +1,35 @@
+package identhash
+
+import (
+	"io"
+
+	"github.com/mleku/manifold/sha256"
+)
+
+const Len = 8
+
+type T struct{ val []byte }
+
+func New() (i *T) { return &T{make([]byte, Len)} }
+
+func (i *T) FromIdent(id []byte) (err error) {
+	i.val = sha256.Sum256Bytes(id)[:Len]
+	return
+}
+
+func (i *T) Bytes() (b []byte) { return i.val }
+
+func (i *T) MarshalWrite(w io.Writer) (err error) {
+	_, err = w.Write(i.val)
+	return
+}
+
+func (i *T) UnmarshalRead(r io.Reader) (err error) {
+	if len(i.val) < Len {
+		i.val = make([]byte, Len)
+	} else {
+		i.val = i.val[:Len]
+	}
+	_, err = r.Read(i.val)
+	return
+}

database/keys/types/idhash/idhash.go

@@ -0,0 +1,68 @@
+package idhash
+
+import (
+	"encoding/base64"
+	"io"
+
+	"github.com/mleku/manifold/sha256"
+
+	"github.com/mleku/manifold/chk"
+	"github.com/mleku/manifold/errorf"
+	"github.com/mleku/manifold/hex"
+)
+
+const Len = 8
+
+type T struct{ val []byte }
+
+func New() (i *T) { return &T{make([]byte, Len)} }
+
+func (i *T) FromId(id []byte) (err error) {
+	if len(id) != sha256.Size {
+		err = errorf.E("invalid Id length, got %d require %d", len(id), sha256.Size)
+		return
+	}
+	i.val = sha256.Sum256Bytes(id)[:Len]
+	return
+}
+
+func (i *T) FromIdBase64(idb64 string) (err error) {
+	id := make([]byte, 43)
+	if _, err = base64.RawURLEncoding.Decode(id, []byte(idb64)); chk.E(err) {
+		return
+	}
+	i.val = sha256.Sum256Bytes(id)[:Len]
+	return
+
+}
+
+func (i *T) FromIdHex(idh string) (err error) {
+	var id []byte
+	if id, err = hex.Dec(idh); chk.E(err) {
+		return
+	}
+	if len(id) != sha256.Size {
+		err = errorf.E("invalid Id length, got %d require %d", len(id), sha256.Size)
+		return
+	}
+	i.val = sha256.Sum256Bytes(id)[:Len]
+	return
+
+}
+
+func (i *T) Bytes() (b []byte) { return i.val }
+
+func (i *T) MarshalWrite(w io.Writer) (err error) {
+	_, err = w.Write(i.val)
+	return
+}
+
+func (i *T) UnmarshalRead(r io.Reader) (err error) {
+	if len(i.val) < Len {
+		i.val = make([]byte, Len)
+	} else {
+		i.val = i.val[:Len]
+	}
+	_, err = r.Read(i.val)
+	return
+}

database/keys/types/letter/letter.go

@@ -0,0 +1,29 @@
+package letter
+
+import (
+	"io"
+)
+
+const Len = 1
+
+type T struct {
+	val []byte
+}
+
+func New(letter byte) (p *T) { return &T{[]byte{letter}} }
+
+func (p *T) Set(lb byte) { p.val = []byte{lb} }
+
+func (p *T) Letter() byte { return p.val[0] }
+
+func (p *T) MarshalWrite(w io.Writer) (err error) {
+	_, err = w.Write(p.val)
+	return
+}
+
+func (p *T) UnmarshalRead(r io.Reader) (err error) {
+	val := make([]byte, 1)
+	_, err = r.Read(val)
+	p.val = val
+	return
+}

database/keys/types/number/uint16.go

@@ -0,0 +1,115 @@
+package number
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// Uint16 is a codec for encoding and decoding 16-bit unsigned integers.
+type Uint16 struct {
+	value uint16
+}
+
+// Set sets the value as a uint16.
+func (c *Uint16) Set(value uint16) {
+	c.value = value
+}
+
+// Get gets the value as a uint16.
+func (c *Uint16) Get() uint16 {
+	return c.value
+}
+
+// SetInt sets the value as an int, converting it to uint16. Truncates values outside uint16 range (0-65535).
+func (c *Uint16) SetInt(value int) {
+	c.value = uint16(value)
+}
+
+// GetInt gets the value as an int, converted from uint16.
+func (c *Uint16) GetInt() int {
+	return int(c.value)
+}
+
+// MarshalWrite writes the uint16 value to the provided writer in BigEndian order.
+func (c *Uint16) MarshalWrite(w io.Writer) error {
+	return binary.Write(w, binary.BigEndian, c.value)
+}
+
+// UnmarshalRead reads a uint16 value from the provided reader in BigEndian order.
+func (c *Uint16) UnmarshalRead(r io.Reader) error {
+	return binary.Read(r, binary.BigEndian, &c.value)
+}
+
+type Uint16s []*Uint16
+
+// Union computes the union of the current Uint16s slice with another Uint16s slice. The result
+// contains all unique elements from both slices.
+func (s Uint16s) Union(other Uint16s) Uint16s {
+	valueMap := make(map[uint16]bool)
+	var result Uint16s
+
+	// Add elements from the current Uint16s slice to the result
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	// Add elements from the other Uint16s slice to the result
+	for _, item := range other {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Intersection computes the intersection of the current Uint16s slice with another Uint16s
+// slice. The result contains only the elements that exist in both slices.
+func (s Uint16s) Intersection(other Uint16s) Uint16s {
+	valueMap := make(map[uint16]bool)
+	var result Uint16s
+
+	// Add all elements from the other Uint16s slice to the map
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Check for common elements in the current Uint16s slice
+	for _, item := range s {
+		val := item.Get()
+		if valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Difference computes the difference of the current Uint16s slice with another Uint16s slice.
+// The result contains only the elements that are in the current slice but not in the other
+// slice.
+func (s Uint16s) Difference(other Uint16s) Uint16s {
+	valueMap := make(map[uint16]bool)
+	var result Uint16s
+
+	// Mark all elements in the other Uint16s slice
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Add elements from the current Uint16s slice that are not in the other Uint16s slice
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}

database/keys/types/number/uint16_test.go

@@ -0,0 +1,123 @@
+package number
+
+import (
+	"bytes"
+	"math"
+	"reflect"
+	"testing"
+
+	"lukechampine.com/frand"
+)
+
+func TestUint16(t *testing.T) {
+	// Helper function to generate random 16-bit integers
+	generateRandomUint16 := func() uint16 {
+		return uint16(frand.Intn(math.MaxUint16)) // math.MaxUint16 == 65535
+	}
+
+	for i := 0; i < 100; i++ { // Run test 100 times for random values
+		// Generate a random value
+		randomUint16 := generateRandomUint16()
+		randomInt := int(randomUint16)
+
+		// Create a new encodedUint16
+		encodedUint16 := new(Uint16)
+
+		// Test UInt16 setter and getter
+		encodedUint16.Set(randomUint16)
+		if encodedUint16.Get() != randomUint16 {
+			t.Fatalf("Get mismatch: got %d, expected %d", encodedUint16.Get(), randomUint16)
+		}
+
+		// Test GetInt setter and getter
+		encodedUint16.SetInt(randomInt)
+		if encodedUint16.GetInt() != randomInt {
+			t.Fatalf("GetInt mismatch: got %d, expected %d", encodedUint16.GetInt(), randomInt)
+		}
+
+		// Test encoding to []byte and decoding back
+		bufEnc := new(bytes.Buffer)
+
+		// MarshalWrite
+		err := encodedUint16.MarshalWrite(bufEnc)
+		if err != nil {
+			t.Fatalf("MarshalWrite failed: %v", err)
+		}
+		encoded := bufEnc.Bytes()
+
+		// Create a copy of encoded bytes before decoding
+		bufDec := bytes.NewBuffer(encoded)
+
+		// Decode back the value
+		decodedUint16 := new(Uint16)
+		err = decodedUint16.UnmarshalRead(bufDec)
+		if err != nil {
+			t.Fatalf("UnmarshalRead failed: %v", err)
+		}
+
+		if decodedUint16.Get() != randomUint16 {
+			t.Fatalf("Decoded value mismatch: got %d, expected %d", decodedUint16.Get(), randomUint16)
+		}
+
+		// Compare encoded bytes to ensure correctness
+		if !bytes.Equal(encoded, bufEnc.Bytes()) {
+			t.Fatalf("Byte encoding mismatch: got %v, expected %v", bufEnc.Bytes(), encoded)
+		}
+	}
+}
+
+func TestUint16sSetOperations(t *testing.T) {
+	// Helper function to create a Uint16 with a specific value
+	createUint16 := func(value uint16) *Uint16 {
+		u := &Uint16{}
+		u.Set(value)
+		return u
+	}
+
+	// Prepare test data
+	a := createUint16(1)
+	b := createUint16(2)
+	c := createUint16(3)
+	d := createUint16(4)
+	e := createUint16(1) // Duplicate of a
+
+	// Define slices
+	set1 := Uint16s{a, b, c}              // [1, 2, 3]
+	set2 := Uint16s{d, e, b}              // [4, 1, 2]
+	expectedUnion := Uint16s{a, b, c, d}  // [1, 2, 3, 4]
+	expectedIntersection := Uint16s{a, b} // [1, 2]
+	expectedDifference := Uint16s{c}      // [3]
+
+	// Test Union
+	t.Run("Union", func(t *testing.T) {
+		result := set1.Union(set2)
+		if !reflect.DeepEqual(getUint16Values(result), getUint16Values(expectedUnion)) {
+			t.Errorf("Union failed: expected %v, got %v", getUint16Values(expectedUnion), getUint16Values(result))
+		}
+	})
+
+	// Test Intersection
+	t.Run("Intersection", func(t *testing.T) {
+		result := set1.Intersection(set2)
+		if !reflect.DeepEqual(getUint16Values(result), getUint16Values(expectedIntersection)) {
+			t.Errorf("Intersection failed: expected %v, got %v", getUint16Values(expectedIntersection), getUint16Values(result))
+		}
+	})
+
+	// Test Difference
+	t.Run("Difference", func(t *testing.T) {
+		result := set1.Difference(set2)
+		if !reflect.DeepEqual(getUint16Values(result), getUint16Values(expectedDifference)) {
+			t.Errorf("Difference failed: expected %v, got %v", getUint16Values(expectedDifference), getUint16Values(result))
+		}
+	})
+}
+
+// Helper function to extract uint64 values from Uint16s
+func getUint16Values(slice Uint16s) []uint16 {
+	var values []uint16
+	for _, item := range slice {
+		values = append(values, item.Get())
+	}
+	return values
+}

database/keys/types/number/uint24.go

@@ -0,0 +1,152 @@
+package number
+
+import (
+	"errors"
+	"io"
+)
+
+// MaxUint24 is the maximum value of a 24-bit unsigned integer: 2^24 - 1.
+const MaxUint24 uint32 = 1<<24 - 1
+
+// Uint24 is a codec for encoding and decoding 24-bit unsigned integers.
+type Uint24 struct {
+	value uint32
+}
+
+// Set sets the value as a 24-bit unsigned integer.
+// If the value exceeds the maximum allowable value for 24 bits, it returns an error.
+func (c *Uint24) Set(value uint32) error {
+	if value > MaxUint24 {
+		return errors.New("value exceeds 24-bit range")
+	}
+	c.value = value
+	return nil
+}
+
+// Get gets the value as a 24-bit unsigned integer.
+func (c *Uint24) Get() uint32 {
+	return c.value
+}
+
+// SetInt sets the value as an int, converting it to a 24-bit unsigned integer.
+// If the value is out of the 24-bit range, it returns an error.
+func (c *Uint24) SetInt(value int) error {
+	if value < 0 || uint32(value) > MaxUint24 {
+		return errors.New("value exceeds 24-bit range")
+	}
+	c.value = uint32(value)
+	return nil
+}
+
+// Int gets the value as an int, converted from the 24-bit unsigned integer.
+func (c *Uint24) Int() int {
+	return int(c.value)
+}
+
+// MarshalWrite encodes the 24-bit unsigned integer and writes it directly to the provided io.Writer.
+// The encoding uses 3 bytes in BigEndian order.
+func (c *Uint24) MarshalWrite(w io.Writer) error {
+	if c.value > MaxUint24 {
+		return errors.New("value exceeds 24-bit range")
+	}
+
+	// Write the 3 bytes (BigEndian order) directly to the writer
+	var buf [3]byte
+	buf[0] = byte((c.value >> 16) & 0xFF) // Most significant byte
+	buf[1] = byte((c.value >> 8) & 0xFF)
+	buf[2] = byte(c.value & 0xFF) // Least significant byte
+
+	_, err := w.Write(buf[:]) // Write all 3 bytes to the writer
+	return err
+}
+
+// UnmarshalRead reads 3 bytes directly from the provided io.Reader and decodes it into a 24-bit unsigned integer.
+func (c *Uint24) UnmarshalRead(r io.Reader) error {
+	// Read 3 bytes directly from the reader
+	var buf [3]byte
+	_, err := io.ReadFull(r, buf[:]) // Ensure exactly 3 bytes are read
+	if err != nil {
+		return err
+	}
+
+	// Decode the 3 bytes into a 24-bit unsigned integer
+	c.value = (uint32(buf[0]) << 16) |
+		(uint32(buf[1]) << 8) |
+		uint32(buf[2])
+
+	return nil
+}
+
+type Uint24s []*Uint24
+
+// Union computes the union of the current Uint24s slice with another Uint24s slice. The result
+// contains all unique elements from both slices.
+func (s Uint24s) Union(other Uint24s) Uint24s {
+	valueMap := make(map[uint32]bool)
+	var result Uint24s
+
+	// Add elements from the current Uint24s slice to the result
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	// Add elements from the other Uint24s slice to the result
+	for _, item := range other {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Intersection computes the intersection of the current Uint24s slice with another Uint24s
+// slice. The result contains only the elements that exist in both slices.
+func (s Uint24s) Intersection(other Uint24s) Uint24s {
+	valueMap := make(map[uint32]bool)
+	var result Uint24s
+
+	// Add all elements from the other Uint24s slice to the map
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Check for common elements in the current Uint24s slice
+	for _, item := range s {
+		val := item.Get()
+		if valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Difference computes the difference of the current Uint24s slice with another Uint24s slice.
+// The result contains only the elements that are in the current slice but not in the other
+// slice.
+func (s Uint24s) Difference(other Uint24s) Uint24s {
+	valueMap := make(map[uint32]bool)
+	var result Uint24s
+
+	// Mark all elements in the other Uint24s slice
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Add elements from the current Uint24s slice that are not in the other Uint24s slice
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}

database/keys/types/number/uint24_test.go

@@ -0,0 +1,124 @@
+package number
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+)
+
+func TestUint24(t *testing.T) {
+	tests := []struct {
+		name        string
+		value       uint32
+		expectedErr bool
+	}{
+		{"Minimum Value", 0, false},
+		{"Maximum Value", MaxUint24, false},
+		{"Value in Range", 8374263, false},           // Example value within the range
+		{"Value Exceeds Range", MaxUint24 + 1, true}, // Exceeds 24-bit limit
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			codec := new(Uint24)
+
+			// Test Set
+			err := codec.Set(tt.value)
+			if tt.expectedErr {
+				if err == nil {
+					t.Errorf("expected error but got none")
+				}
+				return
+			} else if err != nil {
+				t.Errorf("unexpected error: %v", err)
+				return
+			}
+
+			// Test Get getter
+			if codec.Get() != tt.value {
+				t.Errorf("Get mismatch: got %d, expected %d", codec.Get(), tt.value)
+			}
+
+			// Test MarshalWrite and UnmarshalRead
+			buf := new(bytes.Buffer)
+
+			// MarshalWrite directly to the buffer
+			if err := codec.MarshalWrite(buf); err != nil {
+				t.Fatalf("MarshalWrite failed: %v", err)
+			}
+
+			// Validate encoded size is 3 bytes
+			encoded := buf.Bytes()
+			if len(encoded) != 3 {
+				t.Fatalf("encoded size mismatch: got %d bytes, expected 3 bytes", len(encoded))
+			}
+
+			// Decode from the buffer
+			decoded := new(Uint24)
+			if err := decoded.UnmarshalRead(buf); err != nil {
+				t.Fatalf("UnmarshalRead failed: %v", err)
+			}
+
+			// Validate decoded value
+			if decoded.Get() != tt.value {
+				t.Errorf("Decoded value mismatch: got %d, expected %d", decoded.Get(), tt.value)
+			}
+		})
+	}
+}
+
+func TestUint24sSetOperations(t *testing.T) {
+	// Helper function to create a Uint24 with a specific value
+	createUint24 := func(value uint32) *Uint24 {
+		u := &Uint24{}
+		u.Set(value)
+		return u
+	}
+
+	// Prepare test data
+	a := createUint24(1)
+	b := createUint24(2)
+	c := createUint24(3)
+	d := createUint24(4)
+	e := createUint24(1) // Duplicate of a
+
+	// Define slices
+	set1 := Uint24s{a, b, c}              // [1, 2, 3]
+	set2 := Uint24s{d, e, b}              // [4, 1, 2]
+	expectedUnion := Uint24s{a, b, c, d}  // [1, 2, 3, 4]
+	expectedIntersection := Uint24s{a, b} // [1, 2]
+	expectedDifference := Uint24s{c}      // [3]
+
+	// Test Union
+	t.Run("Union", func(t *testing.T) {
+		result := set1.Union(set2)
+		if !reflect.DeepEqual(getUint24Values(result), getUint24Values(expectedUnion)) {
+			t.Errorf("Union failed: expected %v, got %v", getUint24Values(expectedUnion), getUint24Values(result))
+		}
+	})
+
+	// Test Intersection
+	t.Run("Intersection", func(t *testing.T) {
+		result := set1.Intersection(set2)
+		if !reflect.DeepEqual(getUint24Values(result), getUint24Values(expectedIntersection)) {
+			t.Errorf("Intersection failed: expected %v, got %v", getUint24Values(expectedIntersection), getUint24Values(result))
+		}
+	})
+
+	// Test Difference
+	t.Run("Difference", func(t *testing.T) {
+		result := set1.Difference(set2)
+		if !reflect.DeepEqual(getUint24Values(result), getUint24Values(expectedDifference)) {
+			t.Errorf("Difference failed: expected %v, got %v", getUint24Values(expectedDifference), getUint24Values(result))
+		}
+	})
+}
+
+// Helper function to extract uint64 values from Uint24s
+func getUint24Values(slice Uint24s) []uint32 {
+	var values []uint32
+	for _, item := range slice {
+		values = append(values, item.Get())
+	}
+	return values
+}

database/keys/types/number/uint32.go

@@ -0,0 +1,116 @@
+package number
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// Uint32 is a codec for encoding and decoding 32-bit unsigned integers.
+type Uint32 struct {
+	value uint32
+}
+
+// Set sets the value as a uint32.
+func (c *Uint32) Set(value uint32) {
+	c.value = value
+}
+
+// Get gets the value as a uint32.
+func (c *Uint32) Get() uint32 {
+	return c.value
+}
+
+// SetInt sets the value as an int, converting it to uint32.
+// Values outside the range of uint32 (0–4294967295) will be truncated.
+func (c *Uint32) SetInt(value int) {
+	c.value = uint32(value)
+}
+
+// Int gets the value as an int, converted from uint32.
+func (c *Uint32) Int() int {
+	return int(c.value)
+}
+
+// MarshalWrite writes the uint32 value to the provided writer in BigEndian order.
+func (c *Uint32) MarshalWrite(w io.Writer) error {
+	return binary.Write(w, binary.BigEndian, c.value)
+}
+
+// UnmarshalRead reads a uint32 value from the provided reader in BigEndian order.
+func (c *Uint32) UnmarshalRead(r io.Reader) error {
+	return binary.Read(r, binary.BigEndian, &c.value)
+}
+
+type Uint32s []*Uint32
+
+// Union computes the union of the current Uint32s slice with another Uint32s slice. The result
+// contains all unique elements from both slices.
+func (s Uint32s) Union(other Uint32s) Uint32s {
+	valueMap := make(map[uint32]bool)
+	var result Uint32s
+
+	// Add elements from the current Uint32s slice to the result
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	// Add elements from the other Uint32s slice to the result
+	for _, item := range other {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Intersection computes the intersection of the current Uint32s slice with another Uint32s
+// slice. The result contains only the elements that exist in both slices.
+func (s Uint32s) Intersection(other Uint32s) Uint32s {
+	valueMap := make(map[uint32]bool)
+	var result Uint32s
+
+	// Add all elements from the other Uint32s slice to the map
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Check for common elements in the current Uint32s slice
+	for _, item := range s {
+		val := item.Get()
+		if valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Difference computes the difference of the current Uint32s slice with another Uint32s slice.
+// The result contains only the elements that are in the current slice but not in the other
+// slice.
+func (s Uint32s) Difference(other Uint32s) Uint32s {
+	valueMap := make(map[uint32]bool)
+	var result Uint32s
+
+	// Mark all elements in the other Uint32s slice
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Add elements from the current Uint32s slice that are not in the other Uint32s slice
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}

database/keys/types/number/uint32_test.go

@@ -0,0 +1,123 @@
+package number
+
+import (
+	"bytes"
+	"math"
+	"reflect"
+	"testing"
+
+	"lukechampine.com/frand"
+)
+
+func TestUint32(t *testing.T) {
+	// Helper function to generate random 32-bit integers
+	generateRandomUint32 := func() uint32 {
+		return uint32(frand.Intn(math.MaxUint32)) // math.MaxUint32 == 4294967295
+	}
+
+	for i := 0; i < 100; i++ { // Run test 100 times for random values
+		// Generate a random value
+		randomUint32 := generateRandomUint32()
+		randomInt := int(randomUint32)
+
+		// Create a new codec
+		codec := new(Uint32)
+
+		// Test Uint32 setter and getter
+		codec.Set(randomUint32)
+		if codec.Get() != randomUint32 {
+			t.Fatalf("Uint32 mismatch: got %d, expected %d", codec.Get(), randomUint32)
+		}
+
+		// Test GetInt setter and getter
+		codec.SetInt(randomInt)
+		if codec.Int() != randomInt {
+			t.Fatalf("GetInt mismatch: got %d, expected %d", codec.Int(), randomInt)
+		}
+
+		// Test encoding to []byte and decoding back
+		bufEnc := new(bytes.Buffer)
+
+		// MarshalWrite
+		err := codec.MarshalWrite(bufEnc)
+		if err != nil {
+			t.Fatalf("MarshalWrite failed: %v", err)
+		}
+		encoded := bufEnc.Bytes()
+
+		// Create a copy of encoded bytes before decoding
+		bufDec := bytes.NewBuffer(encoded)
+
+		// Decode back the value
+		decoded := new(Uint32)
+		err = decoded.UnmarshalRead(bufDec)
+		if err != nil {
+			t.Fatalf("UnmarshalRead failed: %v", err)
+		}
+
+		if decoded.Get() != randomUint32 {
+			t.Fatalf("Decoded value mismatch: got %d, expected %d", decoded.Get(), randomUint32)
+		}
+
+		// Compare encoded bytes to ensure correctness
+		if !bytes.Equal(encoded, bufEnc.Bytes()) {
+			t.Fatalf("Byte encoding mismatch: got %v, expected %v", bufEnc.Bytes(), encoded)
+		}
+	}
+}
+
+func TestUint32sSetOperations(t *testing.T) {
+	// Helper function to create a Uint32 with a specific value
+	createUint32 := func(value uint32) *Uint32 {
+		u := &Uint32{}
+		u.Set(value)
+		return u
+	}
+
+	// Prepare test data
+	a := createUint32(1)
+	b := createUint32(2)
+	c := createUint32(3)
+	d := createUint32(4)
+	e := createUint32(1) // Duplicate of a
+
+	// Define slices
+	set1 := Uint32s{a, b, c}              // [1, 2, 3]
+	set2 := Uint32s{d, e, b}              // [4, 1, 2]
+	expectedUnion := Uint32s{a, b, c, d}  // [1, 2, 3, 4]
+	expectedIntersection := Uint32s{a, b} // [1, 2]
+	expectedDifference := Uint32s{c}      // [3]
+
+	// Test Union
+	t.Run("Union", func(t *testing.T) {
+		result := set1.Union(set2)
+		if !reflect.DeepEqual(getUint32Values(result), getUint32Values(expectedUnion)) {
+			t.Errorf("Union failed: expected %v, got %v", getUint32Values(expectedUnion), getUint32Values(result))
+		}
+	})
+
+	// Test Intersection
+	t.Run("Intersection", func(t *testing.T) {
+		result := set1.Intersection(set2)
+		if !reflect.DeepEqual(getUint32Values(result), getUint32Values(expectedIntersection)) {
+			t.Errorf("Intersection failed: expected %v, got %v", getUint32Values(expectedIntersection), getUint32Values(result))
+		}
+	})
+
+	// Test Difference
+	t.Run("Difference", func(t *testing.T) {
+		result := set1.Difference(set2)
+		if !reflect.DeepEqual(getUint32Values(result), getUint32Values(expectedDifference)) {
+			t.Errorf("Difference failed: expected %v, got %v", getUint32Values(expectedDifference), getUint32Values(result))
+		}
+	})
+}
+
+// Helper function to extract uint64 values from Uint32s
+func getUint32Values(slice Uint32s) []uint32 {
+	var values []uint32
+	for _, item := range slice {
+		values = append(values, item.Get())
+	}
+	return values
+}

database/keys/types/number/uint40.go

@@ -0,0 +1,149 @@
+package number
+
+import (
+	"errors"
+	"io"
+)
+
+// MaxUint40 is the maximum value of a 40-bit unsigned integer: 2^40 - 1.
+const MaxUint40 uint64 = 1<<40 - 1
+
+// Uint40 is a codec for encoding and decoding 40-bit unsigned integers.
+type Uint40 struct{ value uint64 }
+
+// Set sets the value as a 40-bit unsigned integer.
+// If the value exceeds the maximum allowable value for 40 bits, it returns an error.
+func (c *Uint40) Set(value uint64) error {
+	if value > MaxUint40 {
+		return errors.New("value exceeds 40-bit range")
+	}
+	c.value = value
+	return nil
+}
+
+// Get gets the value as a 40-bit unsigned integer.
+func (c *Uint40) Get() uint64 { return c.value }
+
+// SetInt sets the value as an int, converting it to a 40-bit unsigned integer.
+// If the value is out of the 40-bit range, it returns an error.
+func (c *Uint40) SetInt(value int) error {
+	if value < 0 || uint64(value) > MaxUint40 {
+		return errors.New("value exceeds 40-bit range")
+	}
+	c.value = uint64(value)
+	return nil
+}
+
+// GetInt gets the value as an int, converted from the 40-bit unsigned integer.
+// Note: If the value exceeds the int range, it will be truncated.
+func (c *Uint40) GetInt() int { return int(c.value) }
+
+// MarshalWrite encodes the 40-bit unsigned integer and writes it to the provided writer.
+// The encoding uses 5 bytes in BigEndian order.
+func (c *Uint40) MarshalWrite(w io.Writer) (err error) {
+	if c.value > MaxUint40 {
+		return errors.New("value exceeds 40-bit range")
+	}
+	// Buffer for the 5 bytes
+	buf := make([]byte, 5)
+	// Write the upper 5 bytes (ignoring the most significant 3 bytes of uint64)
+	buf[0] = byte((c.value >> 32) & 0xFF) // Most significant byte
+	buf[1] = byte((c.value >> 24) & 0xFF)
+	buf[2] = byte((c.value >> 16) & 0xFF)
+	buf[3] = byte((c.value >> 8) & 0xFF)
+	buf[4] = byte(c.value & 0xFF) // Least significant byte
+	_, err = w.Write(buf)
+	return err
+}
+
+// UnmarshalRead reads 5 bytes from the provided reader and decodes it into a 40-bit unsigned integer.
+func (c *Uint40) UnmarshalRead(r io.Reader) (err error) {
+	// Buffer for the 5 bytes
+	buf := make([]byte, 5)
+	_, err = r.Read(buf)
+	if err != nil {
+		return err
+	}
+	// Decode the 5 bytes into a 40-bit unsigned integer
+	c.value = (uint64(buf[0]) << 32) |
+		(uint64(buf[1]) << 24) |
+		(uint64(buf[2]) << 16) |
+		(uint64(buf[3]) << 8) |
+		uint64(buf[4])
+
+	return nil
+}
+
+type Uint40s []*Uint40
+
+// Union computes the union of the current Uint40s slice with another Uint40s slice. The result
+// contains all unique elements from both slices.
+func (s Uint40s) Union(other Uint40s) Uint40s {
+	valueMap := make(map[uint64]bool)
+	var result Uint40s
+
+	// Add elements from the current Uint40s slice to the result
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	// Add elements from the other Uint40s slice to the result
+	for _, item := range other {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Intersection computes the intersection of the current Uint40s slice with another Uint40s
+// slice. The result contains only the elements that exist in both slices.
+func (s Uint40s) Intersection(other Uint40s) Uint40s {
+	valueMap := make(map[uint64]bool)
+	var result Uint40s
+
+	// Add all elements from the other Uint40s slice to the map
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Check for common elements in the current Uint40s slice
+	for _, item := range s {
+		val := item.Get()
+		if valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Difference computes the difference of the current Uint40s slice with another Uint40s slice.
+// The result contains only the elements that are in the current slice but not in the other
+// slice.
+func (s Uint40s) Difference(other Uint40s) Uint40s {
+	valueMap := make(map[uint64]bool)
+	var result Uint40s
+
+	// Mark all elements in the other Uint40s slice
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Add elements from the current Uint40s slice that are not in the other Uint40s slice
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}

database/keys/types/number/uint40_test.go

@@ -0,0 +1,125 @@
+package number
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+)
+
+func TestUint40(t *testing.T) {
+	// Test cases for Get
+	tests := []struct {
+		name        string
+		value       uint64
+		expectedErr bool
+	}{
+		{"Minimum Value", 0, false},
+		{"Maximum Value", MaxUint40, false},
+		{"Value in Range", 109951162777, false}, // Example value within the range
+		{"Value Exceeds Range", MaxUint40 + 1, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			codec := new(Uint40)
+
+			// Test Set
+			err := codec.Set(tt.value)
+			if tt.expectedErr {
+				if err == nil {
+					t.Errorf("expected error but got none")
+				}
+				return
+			} else if err != nil {
+				t.Errorf("unexpected error: %v", err)
+				return
+			}
+
+			// Test Get getter
+			if codec.Get() != tt.value {
+				t.Errorf("Uint40 mismatch: got %d, expected %d", codec.Get(), tt.value)
+			}
+
+			// Test MarshalWrite and UnmarshalRead
+			buf := new(bytes.Buffer)
+
+			// Marshal to a buffer
+			if err = codec.MarshalWrite(buf); err != nil {
+				t.Fatalf("MarshalWrite failed: %v", err)
+			}
+
+			// Validate encoded size is 5 bytes
+			encoded := buf.Bytes()
+			if len(encoded) != 5 {
+				t.Fatalf("encoded size mismatch: got %d bytes, expected 5 bytes", len(encoded))
+			}
+
+			// Decode from the buffer
+			decoded := new(Uint40)
+			if err = decoded.UnmarshalRead(buf); err != nil {
+				t.Fatalf("UnmarshalRead failed: %v", err)
+			}
+
+			// Validate decoded value
+			if decoded.Get() != tt.value {
+				t.Errorf("Decoded value mismatch: got %d, expected %d", decoded.Get(), tt.value)
+			}
+		})
+	}
+}
+
+func TestUint40sSetOperations(t *testing.T) {
+	// Helper function to create a Uint64 with a specific value
+	createUint64 := func(value uint64) *Uint40 {
+		u := &Uint40{}
+		u.Set(value)
+		return u
+	}
+
+	// Prepare test data
+	a := createUint64(1)
+	b := createUint64(2)
+	c := createUint64(3)
+	d := createUint64(4)
+	e := createUint64(1) // Duplicate of a
+
+	// Define slices
+	set1 := Uint40s{a, b, c}              // [1, 2, 3]
+	set2 := Uint40s{d, e, b}              // [4, 1, 2]
+	expectedUnion := Uint40s{a, b, c, d}  // [1, 2, 3, 4]
+	expectedIntersection := Uint40s{a, b} // [1, 2]
+	expectedDifference := Uint40s{c}      // [3]
+
+	// Test Union
+	t.Run("Union", func(t *testing.T) {
+		result := set1.Union(set2)
+		if !reflect.DeepEqual(getUint40Values(result), getUint40Values(expectedUnion)) {
+			t.Errorf("Union failed: expected %v, got %v", getUint40Values(expectedUnion), getUint40Values(result))
+		}
+	})
+
+	// Test Intersection
+	t.Run("Intersection", func(t *testing.T) {
+		result := set1.Intersection(set2)
+		if !reflect.DeepEqual(getUint40Values(result), getUint40Values(expectedIntersection)) {
+			t.Errorf("Intersection failed: expected %v, got %v", getUint40Values(expectedIntersection), getUint40Values(result))
+		}
+	})
+
+	// Test Difference
+	t.Run("Difference", func(t *testing.T) {
+		result := set1.Difference(set2)
+		if !reflect.DeepEqual(getUint40Values(result), getUint40Values(expectedDifference)) {
+			t.Errorf("Difference failed: expected %v, got %v", getUint40Values(expectedDifference), getUint40Values(result))
+		}
+	})
+}
+
+// Helper function to extract uint64 values from Uint40s
+func getUint40Values(slice Uint40s) []uint64 {
+	var values []uint64
+	for _, item := range slice {
+		values = append(values, item.Get())
+	}
+	return values
+}

database/keys/types/number/uint64.go

@@ -0,0 +1,117 @@
+package number
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// Uint64 is a codec for encoding and decoding 64-bit unsigned integers.
+type Uint64 struct {
+	value uint64
+}
+
+// Set sets the value as a uint64.
+func (c *Uint64) Set(value uint64) {
+	c.value = value
+}
+
+// Get gets the value as a uint64.
+func (c *Uint64) Get() uint64 {
+	return c.value
+}
+
+// SetInt sets the value as an int, converting it to uint64.
+// Values outside the range of uint64 are truncated.
+func (c *Uint64) SetInt(value int) {
+	c.value = uint64(value)
+}
+
+// Int gets the value as an int, converted from uint64. May truncate if the value exceeds the
+// range of int.
+func (c *Uint64) Int() int {
+	return int(c.value)
+}
+
+// MarshalWrite writes the uint64 value to the provided writer in BigEndian order.
+func (c *Uint64) MarshalWrite(w io.Writer) error {
+	return binary.Write(w, binary.BigEndian, c.value)
+}
+
+// UnmarshalRead reads a uint64 value from the provided reader in BigEndian order.
+func (c *Uint64) UnmarshalRead(r io.Reader) error {
+	return binary.Read(r, binary.BigEndian, &c.value)
+}
+
+type Uint64s []*Uint64
+
+// Union computes the union of the current Uint64s slice with another Uint64s slice. The result
+// contains all unique elements from both slices.
+func (s Uint64s) Union(other Uint64s) Uint64s {
+	valueMap := make(map[uint64]bool)
+	var result Uint64s
+
+	// Add elements from the current Uint64s slice to the result
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	// Add elements from the other Uint64s slice to the result
+	for _, item := range other {
+		val := item.Get()
+		if !valueMap[val] {
+			valueMap[val] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Intersection computes the intersection of the current Uint64s slice with another Uint64s
+// slice. The result contains only the elements that exist in both slices.
+func (s Uint64s) Intersection(other Uint64s) Uint64s {
+	valueMap := make(map[uint64]bool)
+	var result Uint64s
+
+	// Add all elements from the other Uint64s slice to the map
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Check for common elements in the current Uint64s slice
+	for _, item := range s {
+		val := item.Get()
+		if valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// Difference computes the difference of the current Uint64s slice with another Uint64s slice.
+// The result contains only the elements that are in the current slice but not in the other
+// slice.
+func (s Uint64s) Difference(other Uint64s) Uint64s {
+	valueMap := make(map[uint64]bool)
+	var result Uint64s
+
+	// Mark all elements in the other Uint64s slice
+	for _, item := range other {
+		valueMap[item.Get()] = true
+	}
+
+	// Add elements from the current Uint64s slice that are not in the other Uint64s slice
+	for _, item := range s {
+		val := item.Get()
+		if !valueMap[val] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}

database/keys/types/number/uint64_test.go

@@ -0,0 +1,123 @@
+package number
+
+import (
+	"bytes"
+	"math"
+	"reflect"
+	"testing"
+
+	"lukechampine.com/frand"
+)
+
+func TestUint64(t *testing.T) {
+	// Helper function to generate random 64-bit integers
+	generateRandomUint64 := func() uint64 {
+		return frand.Uint64n(math.MaxUint64) // math.MaxUint64 == 18446744073709551615
+	}
+
+	for i := 0; i < 100; i++ { // Run test 100 times for random values
+		// Generate a random value
+		randomUint64 := generateRandomUint64()
+		randomInt := int(randomUint64)
+
+		// Create a new codec
+		codec := new(Uint64)
+
+		// Test UInt64 setter and getter
+		codec.Set(randomUint64)
+		if codec.Get() != randomUint64 {
+			t.Fatalf("Uint64 mismatch: got %d, expected %d", codec.Get(), randomUint64)
+		}
+
+		// Test GetInt setter and getter
+		codec.SetInt(randomInt)
+		if codec.Int() != randomInt {
+			t.Fatalf("GetInt mismatch: got %d, expected %d", codec.Int(), randomInt)
+		}
+
+		// Test encoding to []byte and decoding back
+		bufEnc := new(bytes.Buffer)
+
+		// MarshalWrite
+		err := codec.MarshalWrite(bufEnc)
+		if err != nil {
+			t.Fatalf("MarshalWrite failed: %v", err)
+		}
+		encoded := bufEnc.Bytes()
+
+		// Create a buffer for decoding
+		bufDec := bytes.NewBuffer(encoded)
+
+		// Decode back the value
+		decoded := new(Uint64)
+		err = decoded.UnmarshalRead(bufDec)
+		if err != nil {
+			t.Fatalf("UnmarshalRead failed: %v", err)
+		}
+
+		if decoded.Get() != randomUint64 {
+			t.Fatalf("Decoded value mismatch: got %d, expected %d", decoded.Get(), randomUint64)
+		}
+
+		// Compare encoded bytes to ensure correctness
+		if !bytes.Equal(encoded, bufEnc.Bytes()) {
+			t.Fatalf("Byte encoding mismatch: got %v, expected %v", bufEnc.Bytes(), encoded)
+		}
+	}
+}
+
+func TestUint64sSetOperations(t *testing.T) {
+	// Helper function to create a Uint64 with a specific value
+	createUint64 := func(value uint64) *Uint64 {
+		u := &Uint64{}
+		u.Set(value)
+		return u
+	}
+
+	// Prepare test data
+	a := createUint64(1)
+	b := createUint64(2)
+	c := createUint64(3)
+	d := createUint64(4)
+	e := createUint64(1) // Duplicate of a
+
+	// Define slices
+	set1 := Uint64s{a, b, c}              // [1, 2, 3]
+	set2 := Uint64s{d, e, b}              // [4, 1, 2]
+	expectedUnion := Uint64s{a, b, c, d}  // [1, 2, 3, 4]
+	expectedIntersection := Uint64s{a, b} // [1, 2]
+	expectedDifference := Uint64s{c}      // [3]
+
+	// Test Union
+	t.Run("Union", func(t *testing.T) {
+		result := set1.Union(set2)
+		if !reflect.DeepEqual(getUint64Values(result), getUint64Values(expectedUnion)) {
+			t.Errorf("Union failed: expected %v, got %v", getUint64Values(expectedUnion), getUint64Values(result))
+		}
+	})
+
+	// Test Intersection
+	t.Run("Intersection", func(t *testing.T) {
+		result := set1.Intersection(set2)
+		if !reflect.DeepEqual(getUint64Values(result), getUint64Values(expectedIntersection)) {
+			t.Errorf("Intersection failed: expected %v, got %v", getUint64Values(expectedIntersection), getUint64Values(result))
+		}
+	})
+
+	// Test Difference
+	t.Run("Difference", func(t *testing.T) {
+		result := set1.Difference(set2)
+		if !reflect.DeepEqual(getUint64Values(result), getUint64Values(expectedDifference)) {
+			t.Errorf("Difference failed: expected %v, got %v", getUint64Values(expectedDifference), getUint64Values(result))
+		}
+	})
+}
+
+// Helper function to extract uint64 values from Uint64s
+func getUint64Values(slice Uint64s) []uint64 {
+	var values []uint64
+	for _, item := range slice {
+		values = append(values, item.Get())
+	}
+	return values
+}

database/keys/types/prefix/prefix.go

@@ -0,0 +1,33 @@
+package prefix
+
+import (
+	"io"
+
+	"github.com/mleku/manifold/database/prefixes"
+)
+
+const Len = 2
+
+type T struct {
+	val []byte
+}
+
+func New(prf ...int) (p *T) {
+	if len(prf) > 0 {
+		return &T{[]byte(prefixes.Prefix(prf[0]))}
+	} else {
+		return &T{[]byte{0, 0}}
+	}
+}
+
+func (p *T) Bytes() (b []byte) { return p.val }
+
+func (p *T) MarshalWrite(w io.Writer) (err error) {
+	_, err = w.Write(p.val)
+	return
+}
+
+func (p *T) UnmarshalRead(r io.Reader) (err error) {
+	_, err = r.Read(p.val)
+	return
+}

database/keys/types/pubhash/pubhash.go

@@ -0,0 +1,58 @@
+package pubhash
+
+import (
+	"io"
+
+	"github.com/mleku/manifold/chk"
+	"github.com/mleku/manifold/ec/schnorr"
+	"github.com/mleku/manifold/errorf"
+	"github.com/mleku/manifold/hex"
+	"github.com/mleku/manifold/sha256"
+)
+
+const Len = 8
+
+type T struct{ val []byte }
+
+func New() (ph *T) { return &T{make([]byte, Len)} }
+
+func (ph *T) FromPubkey(pk []byte) (err error) {
+	if len(pk) != schnorr.PubKeyBytesLen {
+		err = errorf.E("invalid Pubkey length, got %d require %d",
+			len(pk), schnorr.PubKeyBytesLen)
+		return
+	}
+	ph.val = sha256.Sum256Bytes(pk)[:Len]
+	return
+}
+
+func (ph *T) FromPubkeyHex(pk string) (err error) {
+	if len(pk) != schnorr.PubKeyBytesLen*2 {
+		err = errorf.E("invalid Pubkey length, got %d require %d",
+			len(pk), schnorr.PubKeyBytesLen*2)
+		return
+	}
+	var pkb []byte
+	if pkb, err = hex.Dec(pk); chk.E(err) {
+		return
+	}
+	ph.val = sha256.Sum256Bytes(pkb)[:Len]
+	return
+}
+
+func (ph *T) Bytes() (b []byte) { return ph.val }
+
+func (ph *T) MarshalWrite(w io.Writer) (err error) {
+	_, err = w.Write(ph.val)
+	return
+}
+
+func (ph *T) UnmarshalRead(r io.Reader) (err error) {
+	if len(ph.val) < Len {
+		ph.val = make([]byte, Len)
+	} else {
+		ph.val = ph.val[:Len]
+	}
+	_, err = r.Read(ph.val)
+	return
+}

database/keys/types/timestamp/timestamp.go

@@ -0,0 +1,55 @@
+package timestamp
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/mleku/manifold/chk"
+	"github.com/mleku/manifold/database/keys/types/number"
+)
+
+const Len = 8
+
+type T struct{ val int64 }
+
+func (ts *T) FromInt(t int)     { ts.val = int64(t) }
+func (ts *T) FromInt64(t int64) { ts.val = int64(t) }
+
+func FromBytes(timestampBytes []byte) (ts *T, err error) {
+	v := new(number.Uint64)
+	if err = v.UnmarshalRead(bytes.NewBuffer(timestampBytes)); chk.E(err) {
+		return
+	}
+	ts = &T{val: int64(v.Get())}
+	return
+}
+
+func (ts *T) ToTimestamp() (timestamp int64) {
+	return
+}
+func (ts *T) Bytes() (b []byte, err error) {
+	v := new(number.Uint64)
+	buf := new(bytes.Buffer)
+	if err = v.MarshalWrite(buf); chk.E(err) {
+		return
+	}
+	b = buf.Bytes()
+	return
+}
+
+func (ts *T) MarshalWrite(w io.Writer) (err error) {
+	v := new(number.Uint64)
+	if err = v.MarshalWrite(w); chk.E(err) {
+		return
+	}
+	return
+}
+
+func (ts *T) UnmarshalRead(r io.Reader) (err error) {
+	v := new(number.Uint64)
+	if err = v.UnmarshalRead(r); chk.E(err) {
+		return
+	}
+	ts.val = int64(v.Get())
+	return
+}

database/prefixes/prefixes.go

@@ -0,0 +1,87 @@
+package prefixes
+
+import (
+	"io"
+)
+
+const Len = 2
+
+type I string
+
+func (i I) Write(w io.Writer) (n int, err error) { return w.Write([]byte(i)) }
+
+// Prefix returns the two byte human-readable prefixes that go in front of
+// database keys.
+func Prefix(prf int) (i I) {
+	switch prf {
+	case Event:
+		return "ev"
+	case Id:
+		return "id"
+	case IdPubkeyTimestamp:
+		return "fi"
+	case PubkeyTimestamp:
+		return "pt"
+	case Timestamp:
+		return "ts"
+	case TagTimestamp:
+		return "tt"
+	case PubkeyTagTimestamp:
+		return "tp"
+	case FulltextWord:
+		return "fw"
+	}
+	return
+}
+
+// the prefixes.
+//
+// by eliminating kinds and all the categories of nonsense associated with them,
+// the specification becomes a lot simpler. There is no "kind" in manifold; such
+// data would be a tag, like mimetype, and/or encoding.
+
+const (
+	// Event is the whole event stored in binary format
+	//
+	//   [ prefix ][ 8 byte serial ] [ event in binary format ]
+	Event = iota
+
+	// Id contains a truncated 8 byte hash of an event index. This is the
+	// secondary key of an event, the primary key is the serial found in the
+	// Event.
+	//
+	// [ prefix ][ 8 bytes truncated hash of Id ][ 8 serial ]
+	Id
+
+	// IdPubkeyTimestamp is an index designed to enable sorting and filtering of
+	// results found via other indexes, without having to decode the event.
+	//
+	// [ prefix ][ 8 bytes serial ][ 32 bytes full event ID ][ 8 bytes truncated hash of pubkey ][ 8 bytes timestamp ]
+	IdPubkeyTimestamp
+
+	// Timestamp is an index that allows search for the timestamp on the event.
+	//
+	// [ prefix ][ timestamp 8 bytes timestamp ][ 8 serial ]
+	Timestamp
+
+	// PubkeyTimestamp is a composite index that allows search by pubkey
+	// filtered by timestamp.
+	//
+	// [ prefix ][ 8 bytes truncated hash of pubkey ][ 8 bytes timestamp ][ 8 serial ]
+	PubkeyTimestamp
+
+	// PubkeyTagTimestamp allows searching for a pubkey, tag and timestamp.
+	//
+	// [ prefix ][ 8 bytes truncated hash of pubkey ][ 8 bytes truncated hash of key ][ 8 bytes truncated hash of value ][ 8 bytes timestamp ][ 8 serial ]
+	PubkeyTagTimestamp
+
+	// TagTimestamp allows searching for a tag and filter by timestamp.
+	//
+	// [ prefix ][ 8 bytes truncated hash of key ][ 8 bytes truncated hash of value ][ 8 bytes timestamp ][ 8 serial ]
+	TagTimestamp
+
+	// FulltextWord is a fulltext word index, the index contains the whole word.
+	//
+	// [ prefix ][ full word, zero terminated ][ 3 bytes word position in content field ][ 8 serial ]
+	FulltextWord
+)

event/binary.go

@@ -74,17 +74,21 @@ func (e *E) ReadBinary(r io.Reader) (err error) {
 		return
 	}
 	for range vi {
+		// read key length
 		if vi, err = Decode(r); ck(err) {
 			return
 		}
 		key := make([]byte, vi)
+		// read key
 		if _, err = r.Read(key); ck(err) {
 			return
 		}
+		// read value length
 		if vi, err = Decode(r); ck(err) {
 			return
 		}
 		val := make([]byte, vi)
+		// read value
 		if _, err = r.Read(val); ck(err) {
 			return
 		}