use varints for indexes

database/fulltext.go

@@ -10,26 +10,25 @@ import (
 	"x.realy.lol/chk"
 	"x.realy.lol/database/indexes"
 	"x.realy.lol/database/indexes/types/fulltext"
-	"x.realy.lol/database/indexes/types/serial"
+	"x.realy.lol/database/indexes/types/varint"
-	"x.realy.lol/database/indexes/types/size"
 	"x.realy.lol/event"
 	"x.realy.lol/hex"
 	"x.realy.lol/kind"
 )
 
 type Words struct {
-	ser     *serial.S
+	ser     *varint.V
 	ev      *event.E
 	wordMap map[string]int
 }
 
-func (d *D) GetFulltextKeys(ev *event.E, ser *serial.S) (keys [][]byte, err error) {
+func (d *D) GetFulltextKeys(ev *event.E, ser *varint.V) (keys [][]byte, err error) {
 	w := d.GetWordsFromContent(ev)
 	for i := range w {
 		ft := fulltext.New()
 		ft.FromWord([]byte(i))
-		pos := size.New()
+		pos := varint.New()
-		pos.FromUint32(uint32(w[i]))
+		pos.FromInteger(uint64(w[i]))
 		buf := new(bytes.Buffer)
 		if err = indexes.FullTextWordEnc(ft, pos, ser).MarshalWrite(buf); chk.E(err) {
 			return

database/get-event-indexes.go

@@ -12,8 +12,8 @@ import (
 	"x.realy.lol/database/indexes/types/kindidx"
 	"x.realy.lol/database/indexes/types/letter"
 	"x.realy.lol/database/indexes/types/pubhash"
-	"x.realy.lol/database/indexes/types/serial"
 	"x.realy.lol/database/indexes/types/timestamp"
+	"x.realy.lol/database/indexes/types/varint"
 	"x.realy.lol/event"
 	"x.realy.lol/hex"
 	"x.realy.lol/tags"
@@ -21,15 +21,15 @@ import (
 
 // GetEventIndexes generates a set of indexes for a new event record. The first record is the
 // key that should have the binary encoded event as its value.
-func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *serial.S, err error) {
+func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *varint.V, err error) {
 	// log.I.F("getting event indices for\n%s", ev.Serialize())
 	// get a new serial
-	ser = serial.New()
+	ser = varint.New()
 	var s uint64
 	if s, err = d.Serial(); chk.E(err) {
 		return
 	}
-	ser.FromSerial(s)
+	ser.FromInteger(s)
 	// create the event id key
 	id := idhash.New()
 	var idb []byte
@@ -61,7 +61,7 @@ func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *serial.S, err e
 	ca := &timestamp.T{}
 	ca.FromInt64(int64(ev.CreatedAt))
 	evIFiB := new(bytes.Buffer)
-	if err = indexes.FullIndexEnc(fid, p, ki, ca, ser).MarshalWrite(evIFiB); chk.E(err) {
+	if err = indexes.FullIndexEnc(ser, fid, p, ki, ca).MarshalWrite(evIFiB); chk.E(err) {
 		return
 	}
 	indices = append(indices, evIFiB.Bytes())
@@ -97,6 +97,18 @@ func (d *D) GetEventIndexes(ev *event.E) (indices [][]byte, ser *serial.S, err e
 		return
 	}
 	indices = append(indices, evIKiB.Bytes())
+	// Kind index
+	evIKcB := new(bytes.Buffer)
+	if err = indexes.KindCreatedAtEnc(ki, ca, ser).MarshalWrite(evIKcB); chk.E(err) {
+		return
+	}
+	indices = append(indices, evIKcB.Bytes())
+	// Kind index
+	evIKpB := new(bytes.Buffer)
+	if err = indexes.KindPubkeyCreatedAtEnc(ki, p, ca, ser).MarshalWrite(evIKpB); chk.E(err) {
+		return
+	}
+	indices = append(indices, evIKpB.Bytes())
 	// tags
 	// TagA index
 	var atags []tags.Tag_a

database/indexes/indexes.go

@@ -14,9 +14,8 @@ import (
 	"x.realy.lol/database/indexes/types/letter"
 	"x.realy.lol/database/indexes/types/prefix"
 	"x.realy.lol/database/indexes/types/pubhash"
-	"x.realy.lol/database/indexes/types/serial"
-	"x.realy.lol/database/indexes/types/size"
 	"x.realy.lol/database/indexes/types/timestamp"
+	"x.realy.lol/database/indexes/types/varint"
 )
 
 type Encs []codec.I
@@ -49,249 +48,274 @@ func (t *T) UnmarshalRead(r io.Reader) (err error) {
 	return
 }
 
-func EventVars() (ser *serial.S) {
+func EventVars() (ser *varint.V) {
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func EventEnc(ser *serial.S) (enc *T) {
+func EventEnc(ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.Event), ser)
 }
-func EventDec(ser *serial.S) (enc *T) {
+func EventDec(ser *varint.V) (enc *T) {
 	return New(prefix.New(), ser)
 }
 
-func IdVars() (id *idhash.T, ser *serial.S) {
+func IdVars() (id *idhash.T, ser *varint.V) {
 	id = idhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func IdEnc(id *idhash.T, ser *serial.S) (enc *T) {
+func IdEnc(id *idhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.Id), id, ser)
 }
 func IdSearch(id *idhash.T) (enc *T) {
 	return New(prefix.New(prefixes.Id), id)
 }
-func IdDec(id *idhash.T, ser *serial.S) (enc *T) {
+func IdDec(id *idhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), id, ser)
 }
 
-func FullIndexVars() (t *fullid.T, p *pubhash.T, ki *kindidx.T,
+func FullIndexVars() (ser *varint.V, t *fullid.T, p *pubhash.T, ki *kindidx.T,
-	ca *timestamp.T, ser *serial.S) {
+	ca *timestamp.T) {
+	ser = varint.New()
 	t = fullid.New()
 	p = pubhash.New()
 	ki = kindidx.FromKind(0)
 	ca = &timestamp.T{}
-	ser = serial.New()
 	return
 }
-func FullIndexEnc(t *fullid.T, p *pubhash.T, ki *kindidx.T,
+func FullIndexEnc(ser *varint.V, t *fullid.T, p *pubhash.T, ki *kindidx.T,
-	ca *timestamp.T, ser *serial.S) (enc *T) {
+	ca *timestamp.T) (enc *T) {
-	return New(prefix.New(prefixes.FullIndex), t, p, ki, ca, ser)
+	return New(prefix.New(prefixes.FullIndex), ser, t, p, ki, ca)
 }
-func FullIndexDec(t *fullid.T, p *pubhash.T, ki *kindidx.T,
+func FullIndexDec(ser *varint.V, t *fullid.T, p *pubhash.T, ki *kindidx.T,
-	ca *timestamp.T, ser *serial.S) (enc *T) {
+	ca *timestamp.T) (enc *T) {
-	return New(prefix.New(), t, p, ki, ca, ser)
+	return New(prefix.New(), ser, t, p, ki, ca)
 }
 
-func PubkeyVars() (p *pubhash.T, ser *serial.S) {
+func PubkeyVars() (p *pubhash.T, ser *varint.V) {
 	p = pubhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func PubkeyEnc(p *pubhash.T, ser *serial.S) (enc *T) {
+func PubkeyEnc(p *pubhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.Pubkey), p, ser)
 }
-func PubkeyDec(p *pubhash.T, ser *serial.S) (enc *T) {
+func PubkeyDec(p *pubhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), p, ser)
 }
 
-func PubkeyCreatedAtVars() (p *pubhash.T, ca *timestamp.T, ser *serial.S) {
+func PubkeyCreatedAtVars() (p *pubhash.T, ca *timestamp.T, ser *varint.V) {
 	p = pubhash.New()
 	ca = &timestamp.T{}
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func PubkeyCreatedAtEnc(p *pubhash.T, ca *timestamp.T, ser *serial.S) (enc *T) {
+func PubkeyCreatedAtEnc(p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.PubkeyCreatedAt), p, ca, ser)
 }
-func PubkeyCreatedAtDec(p *pubhash.T, ca *timestamp.T, ser *serial.S) (enc *T) {
+func PubkeyCreatedAtDec(p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), p, ca, ser)
 }
 
-func CreatedAtVars() (ca *timestamp.T, ser *serial.S) {
+func CreatedAtVars() (ca *timestamp.T, ser *varint.V) {
 	ca = &timestamp.T{}
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func CreatedAtEnc(ca *timestamp.T, ser *serial.S) (enc *T) {
+func CreatedAtEnc(ca *timestamp.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.CreatedAt), ca, ser)
 }
-func CreatedAtDec(ca *timestamp.T, ser *serial.S) (enc *T) {
+func CreatedAtDec(ca *timestamp.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), ca, ser)
 }
 
-func FirstSeenVars() (ser *serial.S, ts *timestamp.T) {
+func FirstSeenVars() (ser *varint.V, ts *timestamp.T) {
 	ts = &timestamp.T{}
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func FirstSeenEnc(ser *serial.S, ts *timestamp.T) (enc *T) {
+func FirstSeenEnc(ser *varint.V, ts *timestamp.T) (enc *T) {
 	return New(prefix.New(prefixes.FirstSeen), ser, ts)
 }
-func FirstSeenDec(ser *serial.S, ts *timestamp.T) (enc *T) {
+func FirstSeenDec(ser *varint.V, ts *timestamp.T) (enc *T) {
 	return New(prefix.New(), ser, ts)
 }
 
-func KindVars() (ki *kindidx.T, ser *serial.S) {
+func KindVars() (ki *kindidx.T, ser *varint.V) {
 	ki = kindidx.FromKind(0)
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func KindEnc(ki *kindidx.T, ser *serial.S) (enc *T) {
+func KindEnc(ki *kindidx.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.Kind), ki, ser)
 }
-func KindDec(ki *kindidx.T, ser *serial.S) (enc *T) {
+func KindDec(ki *kindidx.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), ki, ser)
 }
 
+func KindCreatedAtVars() (ki *kindidx.T, ca *timestamp.T, ser *varint.V) {
+	ki = kindidx.FromKind(0)
+	ca = &timestamp.T{}
+	ser = varint.New()
+	return
+}
+func KindCreatedAtEnc(ki *kindidx.T, ca *timestamp.T, ser *varint.V) (enc *T) {
+	return New(prefix.New(prefixes.Kind), ki, ca, ser)
+}
+func KindCreatedAtDec(ki *kindidx.T, ca *timestamp.T, ser *varint.V) (enc *T) {
+	return New(prefix.New(), ki, ca, ser)
+}
+
+func KindPubkeyCreatedAtVars() (ki *kindidx.T, p *pubhash.T, ca *timestamp.T, ser *varint.V) {
+	ki = kindidx.FromKind(0)
+	ser = varint.New()
+	return
+}
+func KindPubkeyCreatedAtEnc(ki *kindidx.T, p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
+	return New(prefix.New(prefixes.Kind), ki, p, ca, ser)
+}
+func KindPubkeyCreatedAtDec(ki *kindidx.T, p *pubhash.T, ca *timestamp.T, ser *varint.V) (enc *T) {
+	return New(prefix.New(), ki, p, ca, ser)
+}
+
 type TagA struct {
 	Ki  *kindidx.T
 	P   *pubhash.T
 	Id  *identhash.T
-	Ser *serial.S
+	Ser *varint.V
 }
 
-func TagAVars() (ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) {
+func TagAVars() (ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *varint.V) {
 	ki = kindidx.FromKind(0)
 	p = pubhash.New()
 	id = identhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagAEnc(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) (enc *T) {
+func TagAEnc(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagA), ki, p, id, ser)
 }
-func TagADec(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *serial.S) (enc *T) {
+func TagADec(ki *kindidx.T, p *pubhash.T, id *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), ki, p, id, ser)
 }
 
-func TagEventVars() (id *idhash.T, ser *serial.S) {
+func TagEventVars() (id *idhash.T, ser *varint.V) {
 	id = idhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagEventEnc(id *idhash.T, ser *serial.S) (enc *T) {
+func TagEventEnc(id *idhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagEvent), id, ser)
 }
-func TagEventDec(id *idhash.T, ser *serial.S) (enc *T) {
+func TagEventDec(id *idhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), id, ser)
 }
 
-func TagPubkeyVars() (p *pubhash.T, ser *serial.S) {
+func TagPubkeyVars() (p *pubhash.T, ser *varint.V) {
 	p = pubhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagPubkeyEnc(p *pubhash.T, ser *serial.S) (enc *T) {
+func TagPubkeyEnc(p *pubhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagPubkey), p, ser)
 }
-func TagPubkeyDec(p *pubhash.T, ser *serial.S) (enc *T) {
+func TagPubkeyDec(p *pubhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), p, ser)
 }
 
-func TagHashtagVars() (hashtag *identhash.T, ser *serial.S) {
+func TagHashtagVars() (hashtag *identhash.T, ser *varint.V) {
 	hashtag = identhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagHashtagEnc(hashtag *identhash.T, ser *serial.S) (enc *T) {
+func TagHashtagEnc(hashtag *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagHashtag), hashtag, ser)
 }
-func TagHashtagDec(hashtag *identhash.T, ser *serial.S) (enc *T) {
+func TagHashtagDec(hashtag *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), hashtag, ser)
 }
 
-func TagIdentifierVars() (ident *identhash.T, ser *serial.S) {
+func TagIdentifierVars() (ident *identhash.T, ser *varint.V) {
 	ident = identhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagIdentifierEnc(ident *identhash.T, ser *serial.S) (enc *T) {
+func TagIdentifierEnc(ident *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagIdentifier), ident, ser)
 }
-func TagIdentifierDec(ident *identhash.T, ser *serial.S) (enc *T) {
+func TagIdentifierDec(ident *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), ident, ser)
 }
 
-func TagLetterVars() (l *letter.T, val *identhash.T, ser *serial.S) {
+func TagLetterVars() (l *letter.T, val *identhash.T, ser *varint.V) {
 	l = letter.New(0)
 	val = identhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagLetterEnc(l *letter.T, val *identhash.T, ser *serial.S) (enc *T) {
+func TagLetterEnc(l *letter.T, val *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagLetter), l, val, ser)
 }
-func TagLetterDec(l *letter.T, val *identhash.T, ser *serial.S) (enc *T) {
+func TagLetterDec(l *letter.T, val *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), l, val, ser)
 }
 
-func TagProtectedVars() (p *pubhash.T, ser *serial.S) {
+func TagProtectedVars() (p *pubhash.T, ser *varint.V) {
 	p = pubhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagProtectedEnc(p *pubhash.T, ser *serial.S) (enc *T) {
+func TagProtectedEnc(p *pubhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagProtected), p, ser)
 }
-func TagProtectedDec(p *pubhash.T, ser *serial.S) (enc *T) {
+func TagProtectedDec(p *pubhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), p, ser)
 }
 
-func TagNonstandardVars() (key, value *identhash.T, ser *serial.S) {
+func TagNonstandardVars() (key, value *identhash.T, ser *varint.V) {
 	key = identhash.New()
 	value = identhash.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func TagNonstandardEnc(key, value *identhash.T, ser *serial.S) (enc *T) {
+func TagNonstandardEnc(key, value *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.TagNonstandard), key, value, ser)
 }
-func TagNonstandardDec(key, value *identhash.T, ser *serial.S) (enc *T) {
+func TagNonstandardDec(key, value *identhash.T, ser *varint.V) (enc *T) {
 	return New(prefix.New(), key, value, ser)
 }
 
-func FullTextWordVars() (fw *fulltext.T, pos *size.T, ser *serial.S) {
+func FullTextWordVars() (fw *fulltext.T, pos *varint.V, ser *varint.V) {
 	fw = fulltext.New()
-	pos = size.New()
+	pos = varint.New()
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func FullTextWordEnc(fw *fulltext.T, pos *size.T, ser *serial.S) (enc *T) {
+func FullTextWordEnc(fw *fulltext.T, pos *varint.V, ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.FulltextWord), fw, pos, ser)
 }
-func FullTextWordDec(fw *fulltext.T, pos *size.T, ser *serial.S) (enc *T) {
+func FullTextWordDec(fw *fulltext.T, pos *varint.V, ser *varint.V) (enc *T) {
 	return New(prefix.New(), fw, pos, ser)
 }
 
-func LastAccessedVars() (ser *serial.S) {
+func LastAccessedVars() (ser *varint.V) {
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func LastAccessedEnc(ser *serial.S) (enc *T) {
+func LastAccessedEnc(ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.LastAccessed), ser)
 }
-func LastAccessedDec(ser *serial.S) (enc *T) {
+func LastAccessedDec(ser *varint.V) (enc *T) {
 	return New(prefix.New(), ser)
 }
 
-func AccessCounterVars() (ser *serial.S) {
+func AccessCounterVars() (ser *varint.V) {
-	ser = serial.New()
+	ser = varint.New()
 	return
 }
-func AccessCounterEnc(ser *serial.S) (enc *T) {
+func AccessCounterEnc(ser *varint.V) (enc *T) {
 	return New(prefix.New(prefixes.AccessCounter), ser)
 }
-func AccessCounterDec(ser *serial.S) (enc *T) {
+func AccessCounterDec(ser *varint.V) (enc *T) {
 	return New(prefix.New(), ser)
 }

database/indexes/indexes_test.go

@@ -13,13 +13,14 @@ import (
 	"x.realy.lol/database/indexes/prefixes"
 	"x.realy.lol/database/indexes/types/prefix"
 	"x.realy.lol/ec/schnorr"
+	"x.realy.lol/log"
 )
 
 func TestEvent(t *testing.T) {
 	var err error
 	for range 100 {
 		ser := EventVars()
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		evIdx := EventEnc(ser)
 		evIdx.MarshalWrite(buf)
@@ -30,7 +31,7 @@ func TestEvent(t *testing.T) {
 		if err = evIdx2.UnmarshalRead(buf2); chk.E(err) {
 			t.Fatal(err)
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -58,7 +59,7 @@ func TestId(t *testing.T) {
 		if err = id.FromId(frand.Bytes(sha256.Size)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		evIdx := IdEnc(id, ser)
 		evIdx.MarshalWrite(buf)
@@ -72,7 +73,7 @@ func TestId(t *testing.T) {
 		if !bytes.Equal(id.Bytes(), id2.Bytes()) {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -81,7 +82,7 @@ func TestId(t *testing.T) {
 func TestFullIndex(t *testing.T) {
 	var err error
 	for range 100 {
-		id, p, ki, ca, ser := FullIndexVars()
+		ser, id, p, ki, ca := FullIndexVars()
 		if err = id.FromId(frand.Bytes(sha256.Size)); chk.E(err) {
 			t.Fatal(err)
 		}
@@ -90,20 +91,23 @@ func TestFullIndex(t *testing.T) {
 		}
 		ki.Set(frand.Intn(math.MaxUint16))
 		ca.FromInt64(time.Now().Unix())
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
-		fi := FullIndexEnc(id, p, ki, ca, ser)
+		fi := FullIndexEnc(ser, id, p, ki, ca)
-		fi.MarshalWrite(buf)
+		if err = fi.MarshalWrite(buf); chk.E(err) {
+			t.Fatal(err)
+		}
 		// log.I.S(fi)
 		bin := buf.Bytes()
 		// log.I.S(bin)
 		buf2 := bytes.NewBuffer(bin)
-		id2, p2, ki2, ca2, ser2 := FullIndexVars()
+		ser2, id2, p2, ki2, ca2 := FullIndexVars()
-		fi2 := FullIndexDec(id2, p2, ki2, ca2, ser2)
+		fi2 := FullIndexDec(ser2, id2, p2, ki2, ca2)
 		if err = fi2.UnmarshalRead(buf2); chk.E(err) {
 			t.Fatal(err)
 		}
 		if !bytes.Equal(id.Bytes(), id2.Bytes()) {
+			log.I.S(id, id2)
 			t.Fatal("failed to recover same value as input")
 		}
 		if !bytes.Equal(p.Bytes(), p2.Bytes()) {
@@ -125,7 +129,7 @@ func TestPubkey(t *testing.T) {
 		if err = p.FromPubkey(frand.Bytes(schnorr.PubKeyBytesLen)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := PubkeyEnc(p, ser)
 		fi.MarshalWrite(buf)
@@ -139,7 +143,7 @@ func TestPubkey(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -153,7 +157,7 @@ func TestPubkeyCreatedAt(t *testing.T) {
 			t.Fatal(err)
 		}
 		ca.FromInt64(time.Now().Unix())
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := PubkeyCreatedAtEnc(p, ca, ser)
 		fi.MarshalWrite(buf)
@@ -167,7 +171,7 @@ func TestPubkeyCreatedAt(t *testing.T) {
 		if ca.ToTimestamp() != ca2.ToTimestamp() {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -178,7 +182,7 @@ func TestCreatedAt(t *testing.T) {
 	for range 100 {
 		ca, ser := CreatedAtVars()
 		ca.FromInt64(time.Now().Unix())
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := CreatedAtEnc(ca, ser)
 		fi.MarshalWrite(buf)
@@ -192,7 +196,7 @@ func TestCreatedAt(t *testing.T) {
 		if ca.ToTimestamp() != ca2.ToTimestamp() {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -203,7 +207,7 @@ func TestFirstSeen(t *testing.T) {
 	for range 100 {
 		ser, ts := FirstSeenVars()
 		ts.FromInt64(time.Now().Unix())
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fs := FirstSeenEnc(ser, ts)
 		fs.MarshalWrite(buf)
@@ -214,7 +218,7 @@ func TestFirstSeen(t *testing.T) {
 		if err = fs2.UnmarshalRead(buf2); chk.E(err) {
 			t.Fatal(err)
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 		if ts.ToTimestamp() != ca2.ToTimestamp() {
@@ -228,7 +232,7 @@ func TestKind(t *testing.T) {
 	for range 100 {
 		ki, ser := KindVars()
 		ki.Set(frand.Intn(math.MaxUint16))
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		kIdx := KindEnc(ki, ser)
 		kIdx.MarshalWrite(buf)
@@ -242,7 +246,7 @@ func TestKind(t *testing.T) {
 		if ki.ToKind() != ki2.ToKind() {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -259,7 +263,7 @@ func TestTagA(t *testing.T) {
 			t.Fatal(err)
 		}
 		ki.Set(frand.Intn(math.MaxUint16))
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := TagAEnc(ki, p, id, ser)
 		fi.MarshalWrite(buf)
@@ -279,7 +283,7 @@ func TestTagA(t *testing.T) {
 		if ki.ToKind() != ki2.ToKind() {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -292,7 +296,7 @@ func TestTagEvent(t *testing.T) {
 		if err = id.FromId(frand.Bytes(sha256.Size)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		evIdx := TagEventEnc(id, ser)
 		evIdx.MarshalWrite(buf)
@@ -306,7 +310,7 @@ func TestTagEvent(t *testing.T) {
 		if !bytes.Equal(id.Bytes(), id2.Bytes()) {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -319,7 +323,7 @@ func TestTagPubkey(t *testing.T) {
 		if err = p.FromPubkey(frand.Bytes(schnorr.PubKeyBytesLen)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := TagPubkeyEnc(p, ser)
 		fi.MarshalWrite(buf)
@@ -332,7 +336,7 @@ func TestTagPubkey(t *testing.T) {
 		if err = fi2.UnmarshalRead(buf2); chk.E(err) {
 			t.Fatal(err)
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -345,7 +349,7 @@ func TestTagHashtag(t *testing.T) {
 		if err = id.FromIdent(frand.Bytes(frand.Intn(16) + 8)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := TagHashtagEnc(id, ser)
 		fi.MarshalWrite(buf)
@@ -359,7 +363,7 @@ func TestTagHashtag(t *testing.T) {
 		if !bytes.Equal(id.Bytes(), id2.Bytes()) {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -372,7 +376,7 @@ func TestTagIdentifier(t *testing.T) {
 		if err = id.FromIdent(frand.Bytes(frand.Intn(16) + 8)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := TagIdentifierEnc(id, ser)
 		fi.MarshalWrite(buf)
@@ -386,7 +390,7 @@ func TestTagIdentifier(t *testing.T) {
 		if !bytes.Equal(id.Bytes(), id2.Bytes()) {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -401,7 +405,7 @@ func TestTagLetter(t *testing.T) {
 		}
 		lb := frand.Bytes(1)
 		l.Set(lb[0])
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := TagLetterEnc(l, id, ser)
 		fi.MarshalWrite(buf)
@@ -418,7 +422,7 @@ func TestTagLetter(t *testing.T) {
 		if !bytes.Equal(id.Bytes(), id2.Bytes()) {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -431,7 +435,7 @@ func TestTagProtected(t *testing.T) {
 		if err = p.FromPubkey(frand.Bytes(schnorr.PubKeyBytesLen)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := TagProtectedEnc(p, ser)
 		fi.MarshalWrite(buf)
@@ -445,7 +449,7 @@ func TestTagProtected(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -461,7 +465,7 @@ func TestTagNonstandard(t *testing.T) {
 		if err = v.FromIdent(frand.Bytes(frand.Intn(16) + 8)); chk.E(err) {
 			t.Fatal(err)
 		}
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := TagNonstandardEnc(k, v, ser)
 		fi.MarshalWrite(buf)
@@ -478,7 +482,7 @@ func TestTagNonstandard(t *testing.T) {
 		if !bytes.Equal(v.Bytes(), v2.Bytes()) {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -489,11 +493,13 @@ func TestFulltextWord(t *testing.T) {
 	for range 100 {
 		fw, pos, ser := FullTextWordVars()
 		fw.FromWord(frand.Bytes(frand.Intn(10) + 5))
-		pos.FromUint32(uint32(frand.Intn(math.MaxUint32)))
+		pos.FromInteger(uint64(frand.Intn(math.MaxUint32)))
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := FullTextWordEnc(fw, pos, ser)
-		fi.MarshalWrite(buf)
+		if err = fi.MarshalWrite(buf); chk.E(err) {
+			t.Fatal(err)
+		}
 		bin := buf.Bytes()
 		buf2 := bytes.NewBuffer(bin)
 		fw2, pos2, ser2 := FullTextWordVars()
@@ -507,7 +513,7 @@ func TestFulltextWord(t *testing.T) {
 		if pos.ToUint32() != pos2.ToUint32() {
 			t.Fatal("failed to recover same value as input")
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -517,7 +523,7 @@ func TestLastAccessed(t *testing.T) {
 	var err error
 	for range 100 {
 		ser := LastAccessedVars()
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := LastAccessedEnc(ser)
 		fi.MarshalWrite(buf)
@@ -528,7 +534,7 @@ func TestLastAccessed(t *testing.T) {
 		if err = fi2.UnmarshalRead(buf2); chk.E(err) {
 			t.Fatal(err)
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}
@@ -538,7 +544,7 @@ func TestAccessCounter(t *testing.T) {
 	var err error
 	for range 100 {
 		ser := AccessCounterVars()
-		ser.FromSerial(uint64(frand.Intn(math.MaxInt64)))
+		ser.FromInteger(uint64(frand.Intn(math.MaxInt64)))
 		buf := new(bytes.Buffer)
 		fi := AccessCounterEnc(ser)
 		fi.MarshalWrite(buf)
@@ -549,7 +555,7 @@ func TestAccessCounter(t *testing.T) {
 		if err = fi2.UnmarshalRead(buf2); chk.E(err) {
 			t.Fatal(err)
 		}
-		if ser.ToSerial() != ser2.ToSerial() {
+		if ser.ToUint64() != ser2.ToUint64() {
 			t.Fatal("failed to recover same value as input")
 		}
 	}

database/indexes/prefixes/prefixes.go

@@ -58,6 +58,16 @@ const (
 	// [ prefix ][ 2 bytes kind number ][ 8 serial ]
 	Kind
 
+	// KindCreatedAt is an index of kind and created_at timestamp.
+	//
+	// [ prefix ][ 2 bytes kind number ][ created_at 8 bytes timestamp ][ 8 bytes serial ]
+	KindCreatedAt
+
+	// KindPubkeyCreatedAt is an index of kind and created_at timestamp.
+	//
+	// [ prefix ][ 2 bytes kind number ][ 8 bytes hash of pubkey ][ created_at 8 bytes timestamp ][ 8 bytes serial ]
+	KindPubkeyCreatedAt
+
 	// TagA is an index of `a` tags, which contain kind, pubkey and hash of an arbitrary
 	// text, used to create an abstract reference for a multiplicity of replaceable event with a
 	// kind number. These labels also appear as `d` tags in inbound references, see
@@ -132,43 +142,6 @@ const (
 
 func (i I) Write(w io.Writer) (n int, err error) { return w.Write([]byte(i)) }
 
-// func Identify(r io.Reader) (i int, err error) {
-// 	var prefixes = map[I]int{
-// 		"ev": Event,
-// 		"cf": Config,
-// 		"id": Id,
-// 		"fi": FullIndex,
-// 		"pk": Pubkey,
-// 		"pc": PubkeyCreatedAt,
-// 		"ca": CreatedAt,
-// 		"fs": FirstSeen,
-// 		"ki": Kind,
-// 		"ta": TagA,
-// 		"te": TagEvent,
-// 		"tp": TagPubkey,
-// 		"tt": TagHashtag,
-// 		"td": TagIdentifier,
-// 		"t*": TagLetter,
-// 		"t-": TagProtected,
-// 		"t?": TagNonstandard,
-// 		"fw": FulltextWord,
-// 		"la": LastAccessed,
-// 		"ac": AccessCounter,
-// 	}
-// 	b := make([]byte, Len)
-// 	if _, err = r.Read(b); chk.E(err) {
-// 		return
-// 	}
-// 	s := string(b)
-// 	for ii, v := range prefixes {
-// 		if ii == I(s) {
-// 			return v, nil
-// 		}
-// 	}
-// 	err = errorf.E("no match to known prefix '%s'", s)
-// 	return
-// }
-
 func Prefix(prf int) (i I) {
 	switch prf {
 	case Event:
@@ -189,6 +162,10 @@ func Prefix(prf int) (i I) {
 		return "fs"
 	case Kind:
 		return "ki"
+	case KindCreatedAt:
+		return "kc"
+	case KindPubkeyCreatedAt:
+		return "kp"
 	case TagA:
 		return "ta"
 	case TagEvent:

database/indexes/types/serial/serial.go

@@ -1,50 +0,0 @@
-package serial
-
-import (
-	"encoding/binary"
-	"io"
-
-	"x.realy.lol/errorf"
-)
-
-const Len = 8
-
-type S struct{ val []byte }
-
-func New() (s *S) { return &S{make([]byte, Len)} }
-
-func (s *S) FromSerial(ser uint64) {
-	binary.LittleEndian.PutUint64(s.val, ser)
-	return
-}
-
-func FromBytes(ser []byte) (s *S, err error) {
-	if len(ser) != Len {
-		err = errorf.E("serial must be %d bytes long, got %d", Len, len(ser))
-		return
-	}
-	s = &S{val: ser}
-	return
-}
-
-func (s *S) ToSerial() (ser uint64) {
-	ser = binary.LittleEndian.Uint64(s.val)
-	return
-}
-
-func (s *S) Bytes() (b []byte) { return s.val }
-
-func (s *S) MarshalWrite(w io.Writer) (err error) {
-	_, err = w.Write(s.val)
-	return
-}
-
-func (s *S) UnmarshalRead(r io.Reader) (err error) {
-	if len(s.val) < Len {
-		s.val = make([]byte, Len)
-	} else {
-		s.val = s.val[:Len]
-	}
-	_, err = r.Read(s.val)
-	return
-}

database/indexes/types/size/serial.go

@@ -1,51 +0,0 @@
-package size
-
-import (
-	"encoding/binary"
-	"io"
-
-	"x.realy.lol/errorf"
-)
-
-const Len = 4
-
-type T struct{ val []byte }
-
-func New() (s *T) { return &T{make([]byte, Len)} }
-
-func (s *T) FromUint32(n uint32) {
-	s.val = make([]byte, Len)
-	binary.LittleEndian.PutUint32(s.val, n)
-	return
-}
-
-func FromBytes(val []byte) (s *T, err error) {
-	if len(val) != Len {
-		err = errorf.E("size must be %d bytes long, got %d", Len, len(val))
-		return
-	}
-	s = &T{val: val}
-	return
-}
-
-func (s *T) ToUint32() (ser uint32) {
-	ser = binary.LittleEndian.Uint32(s.val)
-	return
-}
-
-func (s *T) Bytes() (b []byte) { return s.val }
-
-func (s *T) MarshalWrite(w io.Writer) (err error) {
-	_, err = w.Write(s.val)
-	return
-}
-
-func (s *T) UnmarshalRead(r io.Reader) (err error) {
-	if len(s.val) < Len {
-		s.val = make([]byte, Len)
-	} else {
-		s.val = s.val[:Len]
-	}
-	_, err = r.Read(s.val)
-	return
-}

database/indexes/types/varint/varint.go

@@ -0,0 +1,46 @@
+package varint
+
+import (
+	"bytes"
+	"io"
+
+	"x.realy.lol/chk"
+	"x.realy.lol/varint"
+)
+
+type V struct{ val uint64 }
+
+func New() (s *V) { return &V{} }
+
+func (vi *V) FromInteger(ser uint64) {
+	vi.val = ser
+	return
+}
+
+func FromBytes(ser []byte) (s *V, err error) {
+	s = &V{}
+	if s.val, err = varint.Decode(bytes.NewBuffer(ser)); chk.E(err) {
+		return
+	}
+	return
+}
+
+func (vi *V) ToUint64() (ser uint64) { return vi.val }
+
+func (vi *V) ToUint32() (v uint32) { return uint32(vi.val) }
+
+func (vi *V) Bytes() (b []byte) {
+	buf := new(bytes.Buffer)
+	varint.Encode(buf, vi.val)
+	return
+}
+
+func (vi *V) MarshalWrite(w io.Writer) (err error) {
+	varint.Encode(w, vi.val)
+	return
+}
+
+func (vi *V) UnmarshalRead(r io.Reader) (err error) {
+	vi.val, err = varint.Decode(r)
+	return
+}

database/main.go

@@ -35,7 +35,7 @@ func (d *D) Path() string { return d.dataDir }
 // Init sets up the database with the loaded configuration.
 func (d *D) Init(path string) (err error) {
 	d.dataDir = path
-	log.I.Ln("opening realy event store at", d.dataDir)
+	log.I.Ln("opening realy database at", d.dataDir)
 	opts := badger.DefaultOptions(d.dataDir)
 	opts.BlockCacheSize = int64(d.BlockCacheSize)
 	opts.BlockSize = units.Gb

database/store.go

@@ -6,8 +6,8 @@ import (
 
 	"x.realy.lol/chk"
 	"x.realy.lol/database/indexes"
-	"x.realy.lol/database/indexes/types/serial"
 	"x.realy.lol/database/indexes/types/timestamp"
+	"x.realy.lol/database/indexes/types/varint"
 	"x.realy.lol/errorf"
 	"x.realy.lol/event"
 )
@@ -24,7 +24,7 @@ func (d *D) StoreEvent(ev *event.E) (err error) {
 			return
 		}
 	}
-	var ser *serial.S
+	var ser *varint.V
 	var idxs [][]byte
 	if idxs, ser, err = d.GetEventIndexes(ev); chk.E(err) {
 		return
@@ -61,7 +61,7 @@ func (d *D) StoreEvent(ev *event.E) (err error) {
 	if err = indexes.AccessCounterEnc(ser).MarshalWrite(acI); chk.E(err) {
 		return
 	}
-	ac := serial.New()
+	ac := varint.New()
 	if err = d.Set(acI.Bytes(), ac.Bytes()); chk.E(err) {
 		return
 	}