mleku
c1bd05fb04
Some checks failed
Go / build-and-release (push) Has been cancelled
This commit allows skipping authentication, permission checks, and certain filters (e.g., deletions, expirations) when the ACL mode is set to "none" (open relay mode). It also introduces a configuration option to disable query caching to reduce memory usage. These changes improve operational flexibility for open relay setups and resource-constrained environments.
449 lines
14 KiB
Go
449 lines
14 KiB
Go
//go:build !(js && wasm)
|
|
|
|
package database
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/dgraph-io/badger/v4"
|
|
"lol.mleku.dev/chk"
|
|
"lol.mleku.dev/log"
|
|
"next.orly.dev/pkg/database/indexes"
|
|
"next.orly.dev/pkg/database/indexes/types"
|
|
"next.orly.dev/pkg/mode"
|
|
"git.mleku.dev/mleku/nostr/encoders/event"
|
|
"git.mleku.dev/mleku/nostr/encoders/filter"
|
|
"git.mleku.dev/mleku/nostr/encoders/hex"
|
|
"git.mleku.dev/mleku/nostr/encoders/kind"
|
|
"git.mleku.dev/mleku/nostr/encoders/tag"
|
|
)
|
|
|
|
var (
|
|
// ErrOlderThanExisting is returned when a candidate event is older than an existing replaceable/addressable event.
|
|
ErrOlderThanExisting = errors.New("older than existing event")
|
|
// ErrMissingDTag is returned when a parameterized replaceable event lacks the required 'd' tag.
|
|
ErrMissingDTag = errors.New("event is missing a d tag identifier")
|
|
)
|
|
|
|
func (d *D) GetSerialsFromFilter(f *filter.F) (
|
|
sers types.Uint40s, err error,
|
|
) {
|
|
// Try p-tag graph optimization first
|
|
if CanUsePTagGraph(f) {
|
|
log.D.F("GetSerialsFromFilter: trying p-tag graph optimization")
|
|
if sers, err = d.QueryPTagGraph(f); err == nil && len(sers) >= 0 {
|
|
log.D.F("GetSerialsFromFilter: p-tag graph optimization returned %d serials", len(sers))
|
|
return
|
|
}
|
|
// Fall through to traditional indexes on error
|
|
log.D.F("GetSerialsFromFilter: p-tag graph optimization failed, falling back to traditional indexes: %v", err)
|
|
err = nil
|
|
}
|
|
|
|
var idxs []Range
|
|
if idxs, err = GetIndexesFromFilter(f); chk.E(err) {
|
|
return
|
|
}
|
|
// Pre-allocate slice with estimated capacity to reduce reallocations
|
|
sers = make(
|
|
types.Uint40s, 0, len(idxs)*100,
|
|
) // Estimate 100 serials per index
|
|
for _, idx := range idxs {
|
|
var s types.Uint40s
|
|
if s, err = d.GetSerialsByRange(idx); chk.E(err) {
|
|
continue
|
|
}
|
|
sers = append(sers, s...)
|
|
}
|
|
return
|
|
}
|
|
|
|
// WouldReplaceEvent checks if the provided event would replace existing events
|
|
// based on Nostr's replaceable or parameterized replaceable semantics. It
|
|
// returns true if the candidate is newer-or-equal than existing events.
|
|
// If an existing event is newer, it returns (false, nil, ErrOlderThanExisting).
|
|
// If no conflicts exist, it returns (false, nil, nil).
|
|
func (d *D) WouldReplaceEvent(ev *event.E) (bool, types.Uint40s, error) {
|
|
// Only relevant for replaceable or parameterized replaceable kinds
|
|
if !(kind.IsReplaceable(ev.Kind) || kind.IsParameterizedReplaceable(ev.Kind)) {
|
|
return false, nil, nil
|
|
}
|
|
|
|
var f *filter.F
|
|
if kind.IsReplaceable(ev.Kind) {
|
|
f = &filter.F{
|
|
Authors: tag.NewFromBytesSlice(ev.Pubkey),
|
|
Kinds: kind.NewS(kind.New(ev.Kind)),
|
|
}
|
|
} else {
|
|
// parameterized replaceable requires 'd' tag
|
|
dTag := ev.Tags.GetFirst([]byte("d"))
|
|
if dTag == nil {
|
|
return false, nil, ErrMissingDTag
|
|
}
|
|
f = &filter.F{
|
|
Authors: tag.NewFromBytesSlice(ev.Pubkey),
|
|
Kinds: kind.NewS(kind.New(ev.Kind)),
|
|
Tags: tag.NewS(
|
|
tag.NewFromAny("d", dTag.Value()),
|
|
),
|
|
}
|
|
}
|
|
|
|
sers, err := d.GetSerialsFromFilter(f)
|
|
if chk.E(err) {
|
|
return false, nil, err
|
|
}
|
|
if len(sers) == 0 {
|
|
return false, nil, nil
|
|
}
|
|
|
|
// Determine if any existing event is newer than the candidate
|
|
shouldReplace := true
|
|
for _, s := range sers {
|
|
oldEv, ferr := d.FetchEventBySerial(s)
|
|
if chk.E(ferr) {
|
|
continue
|
|
}
|
|
if ev.CreatedAt < oldEv.CreatedAt {
|
|
shouldReplace = false
|
|
break
|
|
}
|
|
}
|
|
if shouldReplace {
|
|
return true, nil, nil
|
|
}
|
|
return false, nil, ErrOlderThanExisting
|
|
}
|
|
|
|
// SaveEvent saves an event to the database, generating all the necessary indexes.
|
|
func (d *D) SaveEvent(c context.Context, ev *event.E) (
|
|
replaced bool, err error,
|
|
) {
|
|
if ev == nil {
|
|
err = errors.New("nil event")
|
|
return
|
|
}
|
|
|
|
// Reject ephemeral events (kinds 20000-29999) - they should never be stored
|
|
if ev.Kind >= 20000 && ev.Kind <= 29999 {
|
|
err = errors.New("blocked: ephemeral events should not be stored")
|
|
return
|
|
}
|
|
|
|
// Validate kind 3 (follow list) events have at least one p tag
|
|
// This prevents storing malformed follow lists that may come from buggy relays
|
|
if ev.Kind == 3 {
|
|
hasPTag := false
|
|
tagCount := 0
|
|
if ev.Tags != nil {
|
|
tagCount = ev.Tags.Len()
|
|
for _, tag := range *ev.Tags {
|
|
if tag != nil && tag.Len() >= 2 {
|
|
key := tag.Key()
|
|
if len(key) == 1 && key[0] == 'p' {
|
|
hasPTag = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if !hasPTag {
|
|
log.W.F("SaveEvent: rejecting kind 3 event without p tags from pubkey %x (total tags: %d, event ID: %x)",
|
|
ev.Pubkey, tagCount, ev.ID)
|
|
err = errors.New("blocked: kind 3 follow list events must have at least one p tag")
|
|
return
|
|
}
|
|
}
|
|
|
|
// check if the event already exists
|
|
var ser *types.Uint40
|
|
if ser, err = d.GetSerialById(ev.ID); err == nil && ser != nil {
|
|
err = errors.New("blocked: event already exists: " + hex.Enc(ev.ID[:]))
|
|
return
|
|
}
|
|
|
|
// If the error is "id not found", we can proceed with saving the event
|
|
if err != nil && strings.Contains(err.Error(), "id not found in database") {
|
|
// Reset error since this is expected for new events
|
|
err = nil
|
|
} else if err != nil {
|
|
// For any other error, return it
|
|
// log.E.F("error checking if event exists: %s", err)
|
|
return
|
|
}
|
|
|
|
// Check if the event has been deleted before allowing resubmission
|
|
// Skip deletion check when ACL is "none" (open relay mode)
|
|
if !mode.IsOpen() {
|
|
if err = d.CheckForDeleted(ev, nil); err != nil {
|
|
// log.I.F(
|
|
// "SaveEvent: rejecting resubmission of deleted event ID=%s: %v",
|
|
// hex.Enc(ev.ID), err,
|
|
// )
|
|
err = fmt.Errorf("blocked: %s", err.Error())
|
|
return
|
|
}
|
|
}
|
|
// check for replacement - only validate, don't delete old events
|
|
if kind.IsReplaceable(ev.Kind) || kind.IsParameterizedReplaceable(ev.Kind) {
|
|
var werr error
|
|
if replaced, _, werr = d.WouldReplaceEvent(ev); werr != nil {
|
|
if errors.Is(werr, ErrOlderThanExisting) {
|
|
if kind.IsReplaceable(ev.Kind) {
|
|
err = errors.New("blocked: event is older than existing replaceable event")
|
|
} else {
|
|
err = errors.New("blocked: event is older than existing addressable event")
|
|
}
|
|
return
|
|
}
|
|
if errors.Is(werr, ErrMissingDTag) {
|
|
// keep behavior consistent with previous implementation
|
|
err = ErrMissingDTag
|
|
return
|
|
}
|
|
// any other error
|
|
return
|
|
}
|
|
// Note: replaced flag is kept for compatibility but old events are no longer deleted
|
|
}
|
|
// Get the next sequence number for the event
|
|
var serial uint64
|
|
if serial, err = d.seq.Next(); chk.E(err) {
|
|
return
|
|
}
|
|
// Generate all indexes for the event
|
|
var idxs [][]byte
|
|
if idxs, err = GetIndexesForEvent(ev, serial); chk.E(err) {
|
|
return
|
|
}
|
|
|
|
// Collect all pubkeys for graph: author + p-tags
|
|
// Store with direction indicator: author (0) vs p-tag (1)
|
|
type pubkeyWithDirection struct {
|
|
serial *types.Uint40
|
|
isAuthor bool
|
|
}
|
|
pubkeysForGraph := make(map[string]pubkeyWithDirection)
|
|
|
|
// Add author pubkey
|
|
var authorSerial *types.Uint40
|
|
if authorSerial, err = d.GetOrCreatePubkeySerial(ev.Pubkey); chk.E(err) {
|
|
return
|
|
}
|
|
pubkeysForGraph[hex.Enc(ev.Pubkey)] = pubkeyWithDirection{
|
|
serial: authorSerial,
|
|
isAuthor: true,
|
|
}
|
|
|
|
// Extract p-tag pubkeys using GetAll
|
|
pTags := ev.Tags.GetAll([]byte("p"))
|
|
for _, pTag := range pTags {
|
|
if pTag.Len() >= 2 {
|
|
// Get pubkey from p-tag, handling both binary and hex storage formats
|
|
// ValueHex() returns hex regardless of internal storage format
|
|
var ptagPubkey []byte
|
|
if ptagPubkey, err = hex.Dec(string(pTag.ValueHex())); err == nil && len(ptagPubkey) == 32 {
|
|
pkHex := hex.Enc(ptagPubkey)
|
|
// Skip if already added as author
|
|
if _, exists := pubkeysForGraph[pkHex]; !exists {
|
|
var ptagSerial *types.Uint40
|
|
if ptagSerial, err = d.GetOrCreatePubkeySerial(ptagPubkey); chk.E(err) {
|
|
return
|
|
}
|
|
pubkeysForGraph[pkHex] = pubkeyWithDirection{
|
|
serial: ptagSerial,
|
|
isAuthor: false,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// log.T.F(
|
|
// "SaveEvent: generated %d indexes for event %x (kind %d)", len(idxs),
|
|
// ev.ID, ev.Kind,
|
|
// )
|
|
|
|
// Create serial resolver for compact encoding
|
|
resolver := NewDatabaseSerialResolver(d, d.serialCache)
|
|
|
|
// Serialize event in compact format using serial references
|
|
// This dramatically reduces storage by replacing 32-byte IDs/pubkeys with 5-byte serials
|
|
compactData, compactErr := MarshalCompactEvent(ev, resolver)
|
|
|
|
// Calculate legacy size for comparison (for metrics tracking)
|
|
// We marshal to get accurate size comparison
|
|
legacyBuf := new(bytes.Buffer)
|
|
ev.MarshalBinary(legacyBuf)
|
|
legacySize := legacyBuf.Len()
|
|
|
|
if compactErr != nil {
|
|
// Fall back to legacy format if compact encoding fails
|
|
log.W.F("SaveEvent: compact encoding failed, using legacy format: %v", compactErr)
|
|
compactData = legacyBuf.Bytes()
|
|
} else {
|
|
// Track storage savings
|
|
TrackCompactSaving(legacySize, len(compactData))
|
|
log.T.F("SaveEvent: compact %d bytes vs legacy %d bytes (saved %d bytes, %.1f%%)",
|
|
len(compactData), legacySize, legacySize-len(compactData),
|
|
float64(legacySize-len(compactData))/float64(legacySize)*100.0)
|
|
}
|
|
|
|
// Start a transaction to save the event and all its indexes
|
|
err = d.Update(
|
|
func(txn *badger.Txn) (err error) {
|
|
// Pre-allocate key buffer to avoid allocations in loop
|
|
ser := new(types.Uint40)
|
|
if err = ser.Set(serial); chk.E(err) {
|
|
return
|
|
}
|
|
|
|
// Save each index
|
|
for _, key := range idxs {
|
|
if err = txn.Set(key, nil); chk.E(err) {
|
|
return
|
|
}
|
|
}
|
|
|
|
// Store the SerialEventId mapping (serial -> full 32-byte event ID)
|
|
// This is required for reconstructing compact events
|
|
if err = d.StoreEventIdSerial(txn, serial, ev.ID); chk.E(err) {
|
|
return
|
|
}
|
|
|
|
// Cache the event ID mapping
|
|
d.serialCache.CacheEventId(serial, ev.ID)
|
|
|
|
// Store compact event with cmp prefix
|
|
// Format: cmp|serial|compact_event_data
|
|
// This is the only storage format - legacy evt/sev/aev/rev prefixes
|
|
// are handled by migration and no longer written for new events
|
|
cmpKeyBuf := new(bytes.Buffer)
|
|
if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKeyBuf); chk.E(err) {
|
|
return
|
|
}
|
|
if err = txn.Set(cmpKeyBuf.Bytes(), compactData); chk.E(err) {
|
|
return
|
|
}
|
|
|
|
// Create graph edges between event and all related pubkeys
|
|
// This creates bidirectional edges: event->pubkey and pubkey->event
|
|
// Include the event kind and direction for efficient graph queries
|
|
eventKind := new(types.Uint16)
|
|
eventKind.Set(ev.Kind)
|
|
|
|
for _, pkInfo := range pubkeysForGraph {
|
|
// Determine direction for forward edge (event -> pubkey perspective)
|
|
directionForward := new(types.Letter)
|
|
// Determine direction for reverse edge (pubkey -> event perspective)
|
|
directionReverse := new(types.Letter)
|
|
|
|
if pkInfo.isAuthor {
|
|
// Event author relationship
|
|
directionForward.Set(types.EdgeDirectionAuthor) // 0: author
|
|
directionReverse.Set(types.EdgeDirectionAuthor) // 0: is author of event
|
|
} else {
|
|
// P-tag relationship
|
|
directionForward.Set(types.EdgeDirectionPTagOut) // 1: event references pubkey (outbound)
|
|
directionReverse.Set(types.EdgeDirectionPTagIn) // 2: pubkey is referenced (inbound)
|
|
}
|
|
|
|
// Create event -> pubkey edge (with kind and direction)
|
|
epgKeyBuf := new(bytes.Buffer)
|
|
if err = indexes.EventPubkeyGraphEnc(ser, pkInfo.serial, eventKind, directionForward).MarshalWrite(epgKeyBuf); chk.E(err) {
|
|
return
|
|
}
|
|
// Make a copy of the key bytes to avoid buffer reuse issues in txn
|
|
epgKey := make([]byte, epgKeyBuf.Len())
|
|
copy(epgKey, epgKeyBuf.Bytes())
|
|
if err = txn.Set(epgKey, nil); chk.E(err) {
|
|
return
|
|
}
|
|
|
|
// Create pubkey -> event edge (reverse, with kind and direction for filtering)
|
|
pegKeyBuf := new(bytes.Buffer)
|
|
if err = indexes.PubkeyEventGraphEnc(pkInfo.serial, eventKind, directionReverse, ser).MarshalWrite(pegKeyBuf); chk.E(err) {
|
|
return
|
|
}
|
|
if err = txn.Set(pegKeyBuf.Bytes(), nil); chk.E(err) {
|
|
return
|
|
}
|
|
}
|
|
|
|
// Create event-to-event graph edges for e-tags
|
|
// This enables thread traversal and finding replies/reactions to events
|
|
eTags := ev.Tags.GetAll([]byte("e"))
|
|
for _, eTag := range eTags {
|
|
if eTag.Len() >= 2 {
|
|
// Get event ID from e-tag, handling both binary and hex storage formats
|
|
var targetEventID []byte
|
|
if targetEventID, err = hex.Dec(string(eTag.ValueHex())); err != nil || len(targetEventID) != 32 {
|
|
continue
|
|
}
|
|
|
|
// Look up the target event's serial (if it exists in our database)
|
|
var targetSerial *types.Uint40
|
|
if targetSerial, err = d.GetSerialById(targetEventID); err != nil {
|
|
// Target event not in our database - skip edge creation
|
|
// This is normal for replies to events we don't have
|
|
err = nil
|
|
continue
|
|
}
|
|
|
|
// Create forward edge: source event -> target event (outbound e-tag)
|
|
directionOut := new(types.Letter)
|
|
directionOut.Set(types.EdgeDirectionETagOut)
|
|
eegKeyBuf := new(bytes.Buffer)
|
|
if err = indexes.EventEventGraphEnc(ser, targetSerial, eventKind, directionOut).MarshalWrite(eegKeyBuf); chk.E(err) {
|
|
return
|
|
}
|
|
// Make a copy of the key bytes to avoid buffer reuse issues in txn
|
|
eegKey := make([]byte, eegKeyBuf.Len())
|
|
copy(eegKey, eegKeyBuf.Bytes())
|
|
if err = txn.Set(eegKey, nil); chk.E(err) {
|
|
return
|
|
}
|
|
|
|
// Create reverse edge: target event -> source event (inbound e-tag)
|
|
directionIn := new(types.Letter)
|
|
directionIn.Set(types.EdgeDirectionETagIn)
|
|
geeKeyBuf := new(bytes.Buffer)
|
|
if err = indexes.GraphEventEventEnc(targetSerial, eventKind, directionIn, ser).MarshalWrite(geeKeyBuf); chk.E(err) {
|
|
return
|
|
}
|
|
if err = txn.Set(geeKeyBuf.Bytes(), nil); chk.E(err) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
},
|
|
)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// Process deletion events to actually delete the referenced events
|
|
if ev.Kind == kind.Deletion.K {
|
|
if err = d.ProcessDelete(ev, nil); chk.E(err) {
|
|
log.W.F("failed to process deletion for event %x: %v", ev.ID, err)
|
|
// Don't return error - the deletion event was saved successfully
|
|
err = nil
|
|
}
|
|
}
|
|
|
|
// Invalidate query cache since a new event was stored
|
|
// This ensures subsequent queries will see the new event
|
|
if d.queryCache != nil {
|
|
d.queryCache.Invalidate()
|
|
// log.T.F("SaveEvent: invalidated query cache")
|
|
}
|
|
|
|
return
|
|
}
|