mleku/x-realy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use varints for indexes

Browse Source
This commit is contained in:
mleku
2025-06-10 21:31:24 +01:00
parent bff08415e0
commit faf3ebfdba
10 changed files with 245 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
database/fulltext.go
View File

@@ -10,26 +10,25 @@ import (
"x.realy.lol/chk"
"x.realy.lol/database/indexes"
"x.realy.lol/database/indexes/types/fulltext"
"x.realy.lol/database/indexes/types/serial"
"x.realy.lol/database/indexes/types/size"
"x.realy.lol/database/indexes/types/varint"
"x.realy.lol/event"
"x.realy.lol/hex"
"x.realy.lol/kind"
)
type Words struct {
ser *serial.S
ser *varint.V
ev *event.E
wordMap map[string]int
}
func (d *D) GetFulltextKeys(ev *event.E, ser *serial.S) (keys [][]byte, err error) {
func (d *D) GetFulltextKeys(ev *event.E, ser *varint.V) (keys [][]byte, err error) {
w := d.GetWordsFromContent(ev)
for i := range w {
ft := fulltext.New()
ft.FromWord([]byte(i))
pos := size.New()
pos.FromUint32(uint32(w[i]))
pos := varint.New()
pos.FromInteger(uint64(w[i]))
buf := new(bytes.Buffer)
if err = indexes.FullTextWordEnc(ft, pos, ser).MarshalWrite(buf); chk.E(err) {
return

22
database/get-event-indexes.go
View File

@@ -12,8 +12,8 @@ import (
"x.realy.lol/database/indexes/types/kindidx"
"x.realy.lol/database/indexes/types/letter"
"x.realy.lol/database/indexes/types/pubhash"
"x.realy.lol/database/indexes/types/serial"
"x.realy.lol/database/indexes/types/timestamp"
"x.realy.lol/database/indexes/types/varint"
"x.realy.lol/event"
"x.realy.lol/hex"
"x.realy.lol/tags"
@@ -21,15 +21,15 @@ import (
// GetEventIndexes generates a set of indexes for a new event record. The first record is the
// key that should have the binary encoded event as its value.
func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *serial.S, err error) {
func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *varint.V, err error) {
// log.I.F("getting event indices for\n%s", ev.Serialize())
// get a new serial
ser = serial.New()
ser = varint.New()
var s uint64
if s, err = d.Serial(); chk.E(err) {
return
}
ser.FromSerial(s)
ser.FromInteger(s)
// create the event id key
id := idhash.New()
var idb []byte
@@ -61,7 +61,7 @@ func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *serial.S, err e
ca := &timestamp.T{}
ca.FromInt64(int64(ev.CreatedAt))
evIFiB := new(bytes.Buffer)
if err = indexes.FullIndexEnc(fid, p, ki, ca, ser).MarshalWrite(evIFiB); chk.E(err) {
if err = indexes.FullIndexEnc(ser, fid, p, ki, ca).MarshalWrite(evIFiB); chk.E(err) {
return
}
indices = append(indices, evIFiB.Bytes())
@@ -97,6 +97,18 @@ func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *serial.S, err e
return
}
indices = append(indices, evIKiB.Bytes())
// Kind index
evIKcB := new(bytes.Buffer)
if err = indexes.KindCreatedAtEnc(ki, ca, ser).MarshalWrite(evIKcB); chk.E(err) {
return
}
indices = append(indices, evIKcB.Bytes())
// Kind index
evIKpB := new(bytes.Buffer)
if err = indexes.KindPubkeyCreatedAtEnc(ki, p, ca, ser).MarshalWrite(evIKpB); chk.E(err) {
return
}
indices = append(indices, evIKpB.Bytes())
// tags
// TagA index
var atags []tags.Tag_a

194
database/indexes/indexes.go
View File

@@ -14,9 +14,8 @@ import (
"x.realy.lol/database/indexes/types/letter"
"x.realy.lol/database/indexes/types/prefix"
"x.realy.lol/database/indexes/types/pubhash"
"x.realy.lol/database/indexes/types/serial"
"x.realy.lol/database/indexes/types/size"
"x.realy.lol/database/indexes/types/timestamp"
"x.realy.lol/database/indexes/types/varint"
)
type Encs []codec.I
@@ -49,249 +48,274 @@ func (t *T) UnmarshalRead(r io.Reader) (err error) {
return
}
func EventVars() (ser *serial.S) {
ser = serial.New()
func EventVars() (ser *varint.V) {
ser = varint.New()
return
}
func EventEnc(ser *serial.S) (enc *T) {
func EventEnc(ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.Event), ser)
}
func EventDec(ser *serial.S) (enc *T) {
func EventDec(ser *varint.V) (enc *T) {
return New(prefix.New(), ser)
}
func IdVars() (id *idhash.T, ser *serial.S) {
func IdVars() (id *idhash.T, ser *varint.V) {
id = idhash.New()
ser = serial.New()
ser = varint.New()
return
}
func IdEnc(id *idhash.T, ser *serial.S) (enc *T) {
func IdEnc(id *idhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.Id), id, ser)
}
func IdSearch(id *idhash.T) (enc *T) {
return New(prefix.New(prefixes.Id), id)
}
func IdDec(id *idhash.T, ser *serial.S) (enc *T) {
func IdDec(id *idhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), id, ser)
}
func FullIndexVars() (t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T, ser *serial.S) {
func FullIndexVars() (ser *varint.V, t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T) {
ser = varint.New()
t = fullid.New()
p = pubhash.New()
ki = kindidx.FromKind(0)
ca = &timestamp.T{}
ser = serial.New()
return
}
func FullIndexEnc(t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(prefixes.FullIndex), t, p, ki, ca, ser)
func FullIndexEnc(ser *varint.V, t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T) (enc *T) {
return New(prefix.New(prefixes.FullIndex), ser, t, p, ki, ca)
}
func FullIndexDec(t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T, ser *serial.S) (enc *T) {
return New(prefix.New(), t, p, ki, ca, ser)
func FullIndexDec(ser *varint.V, t *fullid.T, p *pubhash.T, ki *kindidx.T,
ca *timestamp.T) (enc *T) {
return New(prefix.New(), ser, t, p, ki, ca)
}
func PubkeyVars() (p *pubhash.T, ser *serial.S) {
func PubkeyVars() (p *pubhash.T, ser *varint.V) {
p = pubhash.New()
ser = serial.New()
ser = varint.New()
return
}
func PubkeyEnc(p *pubhash.T, ser *serial.S) (enc *T) {
func PubkeyEnc(p *pubhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.Pubkey), p, ser)
}
func PubkeyDec(p *pubhash.T, ser *serial.S) (enc *T) {
func PubkeyDec(p *pubhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), p, ser)
}
func PubkeyCreatedAtVars() (p *pubhash.T, ca *timestamp.T, ser *serial.S) {
func PubkeyCreatedAtVars() (p *pubhash.T, ca *timestamp.T, ser *varint.V) {
p = pubhash.New()
ca = &timestamp.T{}
ser = serial.New()
ser = varint.New()
return
}
func PubkeyCreatedAtEnc(p *pubhash.T, ca *timestamp.T, ser *serial.S) (enc *T) {
func PubkeyCreatedAtEnc(p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.PubkeyCreatedAt), p, ca, ser)
}
func PubkeyCreatedAtDec(p *pubhash.T, ca *timestamp.T, ser *serial.S) (enc *T) {
func PubkeyCreatedAtDec(p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(), p, ca, ser)
}
func CreatedAtVars() (ca *timestamp.T, ser *serial.S) {
func CreatedAtVars() (ca *timestamp.T, ser *varint.V) {
ca = &timestamp.T{}
ser = serial.New()
ser = varint.New()
return
}
func CreatedAtEnc(ca *timestamp.T, ser *serial.S) (enc *T) {
func CreatedAtEnc(ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.CreatedAt), ca, ser)
}
func CreatedAtDec(ca *timestamp.T, ser *serial.S) (enc *T) {
func CreatedAtDec(ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(), ca, ser)
}
func FirstSeenVars() (ser *serial.S, ts *timestamp.T) {
func FirstSeenVars() (ser *varint.V, ts *timestamp.T) {
ts = &timestamp.T{}
ser = serial.New()
ser = varint.New()
return
}
func FirstSeenEnc(ser *serial.S, ts *timestamp.T) (enc *T) {
func FirstSeenEnc(ser *varint.V, ts *timestamp.T) (enc *T) {
return New(prefix.New(prefixes.FirstSeen), ser, ts)
}
func FirstSeenDec(ser *serial.S, ts *timestamp.T) (enc *T) {
func FirstSeenDec(ser *varint.V, ts *timestamp.T) (enc *T) {
return New(prefix.New(), ser, ts)
}
func KindVars() (ki *kindidx.T, ser *serial.S) {
func KindVars() (ki *kindidx.T, ser *varint.V) {
ki = kindidx.FromKind(0)
ser = serial.New()
ser = varint.New()
return
}
func KindEnc(ki *kindidx.T, ser *serial.S) (enc *T) {
func KindEnc(ki *kindidx.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.Kind), ki, ser)
}
func KindDec(ki *kindidx.T, ser *serial.S) (enc *T) {
func KindDec(ki *kindidx.T, ser *varint.V) (enc *T) {
return New(prefix.New(), ki, ser)
}
func KindCreatedAtVars() (ki *kindidx.T, ca *timestamp.T, ser *varint.V) {
ki = kindidx.FromKind(0)
ca = &timestamp.T{}
ser = varint.New()
return
}
func KindCreatedAtEnc(ki *kindidx.T, ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.Kind), ki, ca, ser)
}
func KindCreatedAtDec(ki *kindidx.T, ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(), ki, ca, ser)
}
func KindPubkeyCreatedAtVars() (ki *kindidx.T, p *pubhash.T, ca *timestamp.T, ser *varint.V) {
ki = kindidx.FromKind(0)
ser = varint.New()
return
}
func KindPubkeyCreatedAtEnc(ki *kindidx.T, p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.Kind), ki, p, ca, ser)
}
func KindPubkeyCreatedAtDec(ki *kindidx.T, p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
return New(prefix.New(), ki, p, ca, ser)
}
type TagA struct {
Ki *kindidx.T
P *pubhash.T
Id *identhash.T
Ser *serial.S
Ser *varint.V
}
func TagAVars() (ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) {
func TagAVars() (ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *varint.V) {
ki = kindidx.FromKind(0)
p = pubhash.New()
id = identhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagAEnc(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) (enc *T) {
func TagAEnc(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagA), ki, p, id, ser)
}
func TagADec(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) (enc *T) {
func TagADec(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), ki, p, id, ser)
}
func TagEventVars() (id *idhash.T, ser *serial.S) {
func TagEventVars() (id *idhash.T, ser *varint.V) {
id = idhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagEventEnc(id *idhash.T, ser *serial.S) (enc *T) {
func TagEventEnc(id *idhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagEvent), id, ser)
}
func TagEventDec(id *idhash.T, ser *serial.S) (enc *T) {
func TagEventDec(id *idhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), id, ser)
}
func TagPubkeyVars() (p *pubhash.T, ser *serial.S) {
func TagPubkeyVars() (p *pubhash.T, ser *varint.V) {
p = pubhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagPubkeyEnc(p *pubhash.T, ser *serial.S) (enc *T) {
func TagPubkeyEnc(p *pubhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagPubkey), p, ser)
}
func TagPubkeyDec(p *pubhash.T, ser *serial.S) (enc *T) {
func TagPubkeyDec(p *pubhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), p, ser)
}
func TagHashtagVars() (hashtag *identhash.T, ser *serial.S) {
func TagHashtagVars() (hashtag *identhash.T, ser *varint.V) {
hashtag = identhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagHashtagEnc(hashtag *identhash.T, ser *serial.S) (enc *T) {
func TagHashtagEnc(hashtag *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagHashtag), hashtag, ser)
}
func TagHashtagDec(hashtag *identhash.T, ser *serial.S) (enc *T) {
func TagHashtagDec(hashtag *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), hashtag, ser)
}
func TagIdentifierVars() (ident *identhash.T, ser *serial.S) {
func TagIdentifierVars() (ident *identhash.T, ser *varint.V) {
ident = identhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagIdentifierEnc(ident *identhash.T, ser *serial.S) (enc *T) {
func TagIdentifierEnc(ident *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagIdentifier), ident, ser)
}
func TagIdentifierDec(ident *identhash.T, ser *serial.S) (enc *T) {
func TagIdentifierDec(ident *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), ident, ser)
}
func TagLetterVars() (l *letter.T, val *identhash.T, ser *serial.S) {
func TagLetterVars() (l *letter.T, val *identhash.T, ser *varint.V) {
l = letter.New(0)
val = identhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagLetterEnc(l *letter.T, val *identhash.T, ser *serial.S) (enc *T) {
func TagLetterEnc(l *letter.T, val *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagLetter), l, val, ser)
}
func TagLetterDec(l *letter.T, val *identhash.T, ser *serial.S) (enc *T) {
func TagLetterDec(l *letter.T, val *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), l, val, ser)
}
func TagProtectedVars() (p *pubhash.T, ser *serial.S) {
func TagProtectedVars() (p *pubhash.T, ser *varint.V) {
p = pubhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagProtectedEnc(p *pubhash.T, ser *serial.S) (enc *T) {
func TagProtectedEnc(p *pubhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagProtected), p, ser)
}
func TagProtectedDec(p *pubhash.T, ser *serial.S) (enc *T) {
func TagProtectedDec(p *pubhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), p, ser)
}
func TagNonstandardVars() (key, value *identhash.T, ser *serial.S) {
func TagNonstandardVars() (key, value *identhash.T, ser *varint.V) {
key = identhash.New()
value = identhash.New()
ser = serial.New()
ser = varint.New()
return
}
func TagNonstandardEnc(key, value *identhash.T, ser *serial.S) (enc *T) {
func TagNonstandardEnc(key, value *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.TagNonstandard), key, value, ser)
}
func TagNonstandardDec(key, value *identhash.T, ser *serial.S) (enc *T) {
func TagNonstandardDec(key, value *identhash.T, ser *varint.V) (enc *T) {
return New(prefix.New(), key, value, ser)
}
func FullTextWordVars() (fw *fulltext.T, pos *size.T, ser *serial.S) {
func FullTextWordVars() (fw *fulltext.T, pos *varint.V, ser *varint.V) {
fw = fulltext.New()
pos = size.New()
ser = serial.New()
pos = varint.New()
ser = varint.New()
return
}
func FullTextWordEnc(fw *fulltext.T, pos *size.T, ser *serial.S) (enc *T) {
func FullTextWordEnc(fw *fulltext.T, pos *varint.V, ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.FulltextWord), fw, pos, ser)
}
func FullTextWordDec(fw *fulltext.T, pos *size.T, ser *serial.S) (enc *T) {
func FullTextWordDec(fw *fulltext.T, pos *varint.V, ser *varint.V) (enc *T) {
return New(prefix.New(), fw, pos, ser)
}
func LastAccessedVars() (ser *serial.S) {
ser = serial.New()
func LastAccessedVars() (ser *varint.V) {
ser = varint.New()
return
}
func LastAccessedEnc(ser *serial.S) (enc *T) {
func LastAccessedEnc(ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.LastAccessed), ser)
}
func LastAccessedDec(ser *serial.S) (enc *T) {
func LastAccessedDec(ser *varint.V) (enc *T) {
return New(prefix.New(), ser)
}
func AccessCounterVars() (ser *serial.S) {
ser = serial.New()
func AccessCounterVars() (ser *varint.V) {
ser = varint.New()
return
}
func AccessCounterEnc(ser *serial.S) (enc *T) {
func AccessCounterEnc(ser *varint.V) (enc *T) {
return New(prefix.New(prefixes.AccessCounter), ser)
}
func AccessCounterDec(ser *serial.S) (enc *T) {
func AccessCounterDec(ser *varint.V) (enc *T) {
return New(prefix.New(), ser)
}

94
database/indexes/indexes_test.go
View File

@@ -13,13 +13,14 @@ import (
"x.realy.lol/database/indexes/prefixes"
"x.realy.lol/database/indexes/types/prefix"
"x.realy.lol/ec/schnorr"
"x.realy.lol/log"
)
func TestEvent(t *testing.T) {
var err error
for range 100 {
ser := EventVars()
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
evIdx := EventEnc(ser)
evIdx.MarshalWrite(buf)
@@ -30,7 +31,7 @@ func TestEvent(t *testing.T) {
if err = evIdx2.UnmarshalRead(buf2); chk.E(err) {
t.Fatal(err)
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -58,7 +59,7 @@ func TestId(t *testing.T) {
if err = id.FromId(frand.Bytes(sha256.Size)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
evIdx := IdEnc(id, ser)
evIdx.MarshalWrite(buf)
@@ -72,7 +73,7 @@ func TestId(t *testing.T) {
if !bytes.Equal(id.Bytes(), id2.Bytes()) {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -81,7 +82,7 @@ func TestId(t *testing.T) {
func TestFullIndex(t *testing.T) {
var err error
for range 100 {
id, p, ki, ca, ser := FullIndexVars()
ser, id, p, ki, ca := FullIndexVars()
if err = id.FromId(frand.Bytes(sha256.Size)); chk.E(err) {
t.Fatal(err)
}
@@ -90,20 +91,23 @@ func TestFullIndex(t *testing.T) {
}
ki.Set(frand.Intn(math.MaxUint16))
ca.FromInt64(time.Now().Unix())
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := FullIndexEnc(id, p, ki, ca, ser)
fi.MarshalWrite(buf)
fi := FullIndexEnc(ser, id, p, ki, ca)
if err = fi.MarshalWrite(buf); chk.E(err) {
t.Fatal(err)
}
// log.I.S(fi)
bin := buf.Bytes()
// log.I.S(bin)
buf2 := bytes.NewBuffer(bin)
id2, p2, ki2, ca2, ser2 := FullIndexVars()
fi2 := FullIndexDec(id2, p2, ki2, ca2, ser2)
ser2, id2, p2, ki2, ca2 := FullIndexVars()
fi2 := FullIndexDec(ser2, id2, p2, ki2, ca2)
if err = fi2.UnmarshalRead(buf2); chk.E(err) {
t.Fatal(err)
}
if !bytes.Equal(id.Bytes(), id2.Bytes()) {
log.I.S(id, id2)
t.Fatal("failed to recover same value as input")
}
if !bytes.Equal(p.Bytes(), p2.Bytes()) {
@@ -125,7 +129,7 @@ func TestPubkey(t *testing.T) {
if err = p.FromPubkey(frand.Bytes(schnorr.PubKeyBytesLen)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := PubkeyEnc(p, ser)
fi.MarshalWrite(buf)
@@ -139,7 +143,7 @@ func TestPubkey(t *testing.T) {
t.Fatal(err)
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -153,7 +157,7 @@ func TestPubkeyCreatedAt(t *testing.T) {
t.Fatal(err)
}
ca.FromInt64(time.Now().Unix())
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := PubkeyCreatedAtEnc(p, ca, ser)
fi.MarshalWrite(buf)
@@ -167,7 +171,7 @@ func TestPubkeyCreatedAt(t *testing.T) {
if ca.ToTimestamp() != ca2.ToTimestamp() {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -178,7 +182,7 @@ func TestCreatedAt(t *testing.T) {
for range 100 {
ca, ser := CreatedAtVars()
ca.FromInt64(time.Now().Unix())
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := CreatedAtEnc(ca, ser)
fi.MarshalWrite(buf)
@@ -192,7 +196,7 @@ func TestCreatedAt(t *testing.T) {
if ca.ToTimestamp() != ca2.ToTimestamp() {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -203,7 +207,7 @@ func TestFirstSeen(t *testing.T) {
for range 100 {
ser, ts := FirstSeenVars()
ts.FromInt64(time.Now().Unix())
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fs := FirstSeenEnc(ser, ts)
fs.MarshalWrite(buf)
@@ -214,7 +218,7 @@ func TestFirstSeen(t *testing.T) {
if err = fs2.UnmarshalRead(buf2); chk.E(err) {
t.Fatal(err)
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
if ts.ToTimestamp() != ca2.ToTimestamp() {
@@ -228,7 +232,7 @@ func TestKind(t *testing.T) {
for range 100 {
ki, ser := KindVars()
ki.Set(frand.Intn(math.MaxUint16))
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
kIdx := KindEnc(ki, ser)
kIdx.MarshalWrite(buf)
@@ -242,7 +246,7 @@ func TestKind(t *testing.T) {
if ki.ToKind() != ki2.ToKind() {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -259,7 +263,7 @@ func TestTagA(t *testing.T) {
t.Fatal(err)
}
ki.Set(frand.Intn(math.MaxUint16))
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := TagAEnc(ki, p, id, ser)
fi.MarshalWrite(buf)
@@ -279,7 +283,7 @@ func TestTagA(t *testing.T) {
if ki.ToKind() != ki2.ToKind() {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -292,7 +296,7 @@ func TestTagEvent(t *testing.T) {
if err = id.FromId(frand.Bytes(sha256.Size)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
evIdx := TagEventEnc(id, ser)
evIdx.MarshalWrite(buf)
@@ -306,7 +310,7 @@ func TestTagEvent(t *testing.T) {
if !bytes.Equal(id.Bytes(), id2.Bytes()) {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -319,7 +323,7 @@ func TestTagPubkey(t *testing.T) {
if err = p.FromPubkey(frand.Bytes(schnorr.PubKeyBytesLen)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := TagPubkeyEnc(p, ser)
fi.MarshalWrite(buf)
@@ -332,7 +336,7 @@ func TestTagPubkey(t *testing.T) {
if err = fi2.UnmarshalRead(buf2); chk.E(err) {
t.Fatal(err)
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -345,7 +349,7 @@ func TestTagHashtag(t *testing.T) {
if err = id.FromIdent(frand.Bytes(frand.Intn(16) + 8)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := TagHashtagEnc(id, ser)
fi.MarshalWrite(buf)
@@ -359,7 +363,7 @@ func TestTagHashtag(t *testing.T) {
if !bytes.Equal(id.Bytes(), id2.Bytes()) {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -372,7 +376,7 @@ func TestTagIdentifier(t *testing.T) {
if err = id.FromIdent(frand.Bytes(frand.Intn(16) + 8)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := TagIdentifierEnc(id, ser)
fi.MarshalWrite(buf)
@@ -386,7 +390,7 @@ func TestTagIdentifier(t *testing.T) {
if !bytes.Equal(id.Bytes(), id2.Bytes()) {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -401,7 +405,7 @@ func TestTagLetter(t *testing.T) {
}
lb := frand.Bytes(1)
l.Set(lb[0])
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := TagLetterEnc(l, id, ser)
fi.MarshalWrite(buf)
@@ -418,7 +422,7 @@ func TestTagLetter(t *testing.T) {
if !bytes.Equal(id.Bytes(), id2.Bytes()) {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -431,7 +435,7 @@ func TestTagProtected(t *testing.T) {
if err = p.FromPubkey(frand.Bytes(schnorr.PubKeyBytesLen)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := TagProtectedEnc(p, ser)
fi.MarshalWrite(buf)
@@ -445,7 +449,7 @@ func TestTagProtected(t *testing.T) {
t.Fatal(err)
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -461,7 +465,7 @@ func TestTagNonstandard(t *testing.T) {
if err = v.FromIdent(frand.Bytes(frand.Intn(16) + 8)); chk.E(err) {
t.Fatal(err)
}
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := TagNonstandardEnc(k, v, ser)
fi.MarshalWrite(buf)
@@ -478,7 +482,7 @@ func TestTagNonstandard(t *testing.T) {
if !bytes.Equal(v.Bytes(), v2.Bytes()) {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -489,11 +493,13 @@ func TestFulltextWord(t *testing.T) {
for range 100 {
fw, pos, ser := FullTextWordVars()
fw.FromWord(frand.Bytes(frand.Intn(10) + 5))
pos.FromUint32(uint32(frand.Intn(math.MaxUint32)))
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
pos.FromInteger(uint64(frand.Intn(math.MaxUint32)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := FullTextWordEnc(fw, pos, ser)
fi.MarshalWrite(buf)
if err = fi.MarshalWrite(buf); chk.E(err) {
t.Fatal(err)
}
bin := buf.Bytes()
buf2 := bytes.NewBuffer(bin)
fw2, pos2, ser2 := FullTextWordVars()
@@ -507,7 +513,7 @@ func TestFulltextWord(t *testing.T) {
if pos.ToUint32() != pos2.ToUint32() {
t.Fatal("failed to recover same value as input")
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -517,7 +523,7 @@ func TestLastAccessed(t *testing.T) {
var err error
for range 100 {
ser := LastAccessedVars()
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := LastAccessedEnc(ser)
fi.MarshalWrite(buf)
@@ -528,7 +534,7 @@ func TestLastAccessed(t *testing.T) {
if err = fi2.UnmarshalRead(buf2); chk.E(err) {
t.Fatal(err)
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}
@@ -538,7 +544,7 @@ func TestAccessCounter(t *testing.T) {
var err error
for range 100 {
ser := AccessCounterVars()
ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
buf := new(bytes.Buffer)
fi := AccessCounterEnc(ser)
fi.MarshalWrite(buf)
@@ -549,7 +555,7 @@ func TestAccessCounter(t *testing.T) {
if err = fi2.UnmarshalRead(buf2); chk.E(err) {
t.Fatal(err)
}
if ser.ToSerial() != ser2.ToSerial() {
if ser.ToUint64() != ser2.ToUint64() {
t.Fatal("failed to recover same value as input")
}
}

51
database/indexes/prefixes/prefixes.go
View File

@@ -58,6 +58,16 @@ const (
// [ prefix ][ 2 bytes kind number ][ 8 serial ]
Kind
// KindCreatedAt is an index of kind and created_at timestamp.
//
// [ prefix ][ 2 bytes kind number ][ created_at 8 bytes timestamp ][ 8 bytes serial ]
KindCreatedAt
// KindPubkeyCreatedAt is an index of kind and created_at timestamp.
//
// [ prefix ][ 2 bytes kind number ][ 8 bytes hash of pubkey ][ created_at 8 bytes timestamp ][ 8 bytes serial ]
KindPubkeyCreatedAt
// TagA is an index of `a` tags, which contain kind, pubkey and hash of an arbitrary
// text, used to create an abstract reference for a multiplicity of replaceable event with a
// kind number. These labels also appear as `d` tags in inbound references, see
@@ -132,43 +142,6 @@ const (
func (i I) Write(w io.Writer) (n int, err error) { return w.Write([]byte(i)) }
// func Identify(r io.Reader) (i int, err error) {
// var prefixes = map[I]int{
// "ev": Event,
// "cf": Config,
// "id": Id,
// "fi": FullIndex,
// "pk": Pubkey,
// "pc": PubkeyCreatedAt,
// "ca": CreatedAt,
// "fs": FirstSeen,
// "ki": Kind,
// "ta": TagA,
// "te": TagEvent,
// "tp": TagPubkey,
// "tt": TagHashtag,
// "td": TagIdentifier,
// "t*": TagLetter,
// "t-": TagProtected,
// "t?": TagNonstandard,
// "fw": FulltextWord,
// "la": LastAccessed,
// "ac": AccessCounter,
// }
// b := make([]byte, Len)
// if _, err = r.Read(b); chk.E(err) {
// return
// }
// s := string(b)
// for ii, v := range prefixes {
// if ii == I(s) {
// return v, nil
// }
// }
// err = errorf.E("no match to known prefix '%s'", s)
// return
// }
func Prefix(prf int) (i I) {
switch prf {
case Event:
@@ -189,6 +162,10 @@ func Prefix(prf int) (i I) {
return "fs"
case Kind:
return "ki"
case KindCreatedAt:
return "kc"
case KindPubkeyCreatedAt:
return "kp"
case TagA:
return "ta"
case TagEvent:

50
database/indexes/types/serial/serial.go
View File

@@ -1,50 +0,0 @@
package serial
import (
"encoding/binary"
"io"
"x.realy.lol/errorf"
)
const Len = 8
type S struct{ val []byte }
func New() (s *S) { return &S{make([]byte, Len)} }
func (s *S) FromSerial(ser uint64) {
binary.LittleEndian.PutUint64(s.val, ser)
return
}
func FromBytes(ser []byte) (s *S, err error) {
if len(ser) != Len {
err = errorf.E("serial must be %d bytes long, got %d", Len, len(ser))
return
}
s = &S{val: ser}
return
}
func (s *S) ToSerial() (ser uint64) {
ser = binary.LittleEndian.Uint64(s.val)
return
}
func (s *S) Bytes() (b []byte) { return s.val }
func (s *S) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(s.val)
return
}
func (s *S) UnmarshalRead(r io.Reader) (err error) {
if len(s.val) < Len {
s.val = make([]byte, Len)
} else {
s.val = s.val[:Len]
}
_, err = r.Read(s.val)
return
}

51
database/indexes/types/size/serial.go
View File

@@ -1,51 +0,0 @@
package size
import (
"encoding/binary"
"io"
"x.realy.lol/errorf"
)
const Len = 4
type T struct{ val []byte }
func New() (s *T) { return &T{make([]byte, Len)} }
func (s *T) FromUint32(n uint32) {
s.val = make([]byte, Len)
binary.LittleEndian.PutUint32(s.val, n)
return
}
func FromBytes(val []byte) (s *T, err error) {
if len(val) != Len {
err = errorf.E("size must be %d bytes long, got %d", Len, len(val))
return
}
s = &T{val: val}
return
}
func (s *T) ToUint32() (ser uint32) {
ser = binary.LittleEndian.Uint32(s.val)
return
}
func (s *T) Bytes() (b []byte) { return s.val }
func (s *T) MarshalWrite(w io.Writer) (err error) {
_, err = w.Write(s.val)
return
}
func (s *T) UnmarshalRead(r io.Reader) (err error) {
if len(s.val) < Len {
s.val = make([]byte, Len)
} else {
s.val = s.val[:Len]
}
_, err = r.Read(s.val)
return
}

46
database/indexes/types/varint/varint.go Normal file
View File

@@ -0,0 +1,46 @@
package varint
import (
"bytes"
"io"
"x.realy.lol/chk"
"x.realy.lol/varint"
)
type V struct{ val uint64 }
func New() (s *V) { return &V{} }
func (vi *V) FromInteger(ser uint64) {
vi.val = ser
return
}
func FromBytes(ser []byte) (s *V, err error) {
s = &V{}
if s.val, err = varint.Decode(bytes.NewBuffer(ser)); chk.E(err) {
return
}
return
}
func (vi *V) ToUint64() (ser uint64) { return vi.val }
func (vi *V) ToUint32() (v uint32) { return uint32(vi.val) }
func (vi *V) Bytes() (b []byte) {
buf := new(bytes.Buffer)
varint.Encode(buf, vi.val)
return
}
func (vi *V) MarshalWrite(w io.Writer) (err error) {
varint.Encode(w, vi.val)
return
}
func (vi *V) UnmarshalRead(r io.Reader) (err error) {
vi.val, err = varint.Decode(r)
return
}

2
database/main.go
View File

@@ -35,7 +35,7 @@ func (d *D) Path() string { return d.dataDir }
// Init sets up the database with the loaded configuration.
func (d *D) Init(path string) (err error) {
d.dataDir = path
log.I.Ln("opening realy event store at", d.dataDir)
log.I.Ln("opening realy database at", d.dataDir)
opts := badger.DefaultOptions(d.dataDir)
opts.BlockCacheSize = int64(d.BlockCacheSize)
opts.BlockSize = units.Gb

6
database/store.go
View File

@@ -6,8 +6,8 @@ import (
"x.realy.lol/chk"
"x.realy.lol/database/indexes"
"x.realy.lol/database/indexes/types/serial"
"x.realy.lol/database/indexes/types/timestamp"
"x.realy.lol/database/indexes/types/varint"
"x.realy.lol/errorf"
"x.realy.lol/event"
)
@@ -24,7 +24,7 @@ func (d *D) StoreEvent(ev *event.E) (err error) {
return
}
}
var ser *serial.S
var ser *varint.V
var idxs [][]byte
if idxs, ser, err = d.GetEventIndexes(ev); chk.E(err) {
return
@@ -61,7 +61,7 @@ func (d *D) StoreEvent(ev *event.E) (err error) {
if err = indexes.AccessCounterEnc(ser).MarshalWrite(acI); chk.E(err) {
return
}
ac := serial.New()
ac := varint.New()
if err = d.Set(acI.Bytes(), ac.Bytes()); chk.E(err) {
return
}