503 lines
16 KiB
Go
503 lines
16 KiB
Go
package database
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"lol.mleku.dev/chk"
|
|
"lol.mleku.dev/log"
|
|
"next.orly.dev/pkg/crypto/sha256"
|
|
"next.orly.dev/pkg/database/indexes/types"
|
|
"next.orly.dev/pkg/encoders/event"
|
|
"next.orly.dev/pkg/encoders/filter"
|
|
"next.orly.dev/pkg/encoders/hex"
|
|
"next.orly.dev/pkg/encoders/ints"
|
|
"next.orly.dev/pkg/encoders/kind"
|
|
"next.orly.dev/pkg/encoders/tag"
|
|
"next.orly.dev/pkg/interfaces/store"
|
|
"next.orly.dev/pkg/utils"
|
|
)
|
|
|
|
func CheckExpiration(ev *event.E) (expired bool) {
|
|
var err error
|
|
expTag := ev.Tags.GetFirst([]byte("expiration"))
|
|
if expTag != nil {
|
|
expTS := ints.New(0)
|
|
if _, err = expTS.Unmarshal(expTag.Value()); !chk.E(err) {
|
|
if int64(expTS.N) < time.Now().Unix() {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func (d *D) QueryEvents(c context.Context, f *filter.F) (
|
|
evs event.S, err error,
|
|
) {
|
|
// if there is Ids in the query, this overrides anything else
|
|
var expDeletes types.Uint40s
|
|
var expEvs event.S
|
|
if f.Ids != nil && f.Ids.Len() > 0 {
|
|
// for _, id := range f.Ids.T {
|
|
// log.T.F("QueryEvents: looking for ID=%s", hex.Enc(id))
|
|
// }
|
|
// log.T.F("QueryEvents: ids path, count=%d", f.Ids.Len())
|
|
for _, idx := range f.Ids.T {
|
|
// log.T.F("QueryEvents: lookup id=%s", hex.Enc(idx))
|
|
// we know there is only Ids in this, so run the ID query and fetch.
|
|
var ser *types.Uint40
|
|
var idErr error
|
|
if ser, idErr = d.GetSerialById(idx); idErr != nil {
|
|
// Check if this is a "not found" error which is expected for IDs we don't have
|
|
if strings.Contains(idErr.Error(), "id not found in database") {
|
|
// log.T.F(
|
|
// "QueryEvents: ID not found in database: %s",
|
|
// hex.Enc(idx),
|
|
// )
|
|
} else {
|
|
// Log unexpected errors but continue processing other IDs
|
|
// log.E.F(
|
|
// "QueryEvents: error looking up id=%s err=%v",
|
|
// hex.Enc(idx), idErr,
|
|
// )
|
|
}
|
|
continue
|
|
}
|
|
// Check if the serial is nil, which indicates the ID wasn't found
|
|
if ser == nil {
|
|
// log.T.F("QueryEvents: Serial is nil for ID: %s", hex.Enc(idx))
|
|
continue
|
|
}
|
|
// fetch the events
|
|
var ev *event.E
|
|
if ev, err = d.FetchEventBySerial(ser); err != nil {
|
|
// log.T.F(
|
|
// "QueryEvents: fetch by serial failed for id=%s ser=%v err=%v",
|
|
// hex.Enc(idx), ser, err,
|
|
// )
|
|
continue
|
|
}
|
|
// log.T.F(
|
|
// "QueryEvents: found id=%s kind=%d created_at=%d",
|
|
// hex.Enc(ev.ID), ev.Kind, ev.CreatedAt,
|
|
// )
|
|
// check for an expiration tag and delete after returning the result
|
|
if CheckExpiration(ev) {
|
|
log.T.F(
|
|
"QueryEvents: id=%s filtered out due to expiration",
|
|
hex.Enc(ev.ID),
|
|
)
|
|
expDeletes = append(expDeletes, ser)
|
|
expEvs = append(expEvs, ev)
|
|
continue
|
|
}
|
|
// skip events that have been deleted by a proper deletion event
|
|
if derr := d.CheckForDeleted(ev, nil); derr != nil {
|
|
// log.T.F(
|
|
// "QueryEvents: id=%s filtered out due to deletion: %v",
|
|
// hex.Enc(ev.ID), derr,
|
|
// )
|
|
continue
|
|
}
|
|
// log.T.F(
|
|
// "QueryEvents: id=%s SUCCESSFULLY FOUND, adding to results",
|
|
// hex.Enc(ev.ID),
|
|
// )
|
|
evs = append(evs, ev)
|
|
}
|
|
// sort the events by timestamp
|
|
sort.Slice(
|
|
evs, func(i, j int) bool {
|
|
return evs[i].CreatedAt > evs[j].CreatedAt
|
|
},
|
|
)
|
|
} else {
|
|
// non-IDs path
|
|
var idPkTs []*store.IdPkTs
|
|
// if f.Authors != nil && f.Authors.Len() > 0 && f.Kinds != nil && f.Kinds.Len() > 0 {
|
|
// log.T.F("QueryEvents: authors+kinds path, authors=%d kinds=%d", f.Authors.Len(), f.Kinds.Len())
|
|
// }
|
|
if idPkTs, err = d.QueryForIds(c, f); chk.E(err) {
|
|
return
|
|
}
|
|
// log.T.F("QueryEvents: QueryForIds returned %d candidates", len(idPkTs))
|
|
// Create a map to store the latest version of replaceable events
|
|
replaceableEvents := make(map[string]*event.E)
|
|
// Create a map to store the latest version of parameterized replaceable
|
|
// events
|
|
paramReplaceableEvents := make(map[string]map[string]*event.E)
|
|
// Regular events that are not replaceable
|
|
var regularEvents event.S
|
|
// Map to track deletion events by kind and pubkey (for replaceable
|
|
// events)
|
|
deletionsByKindPubkey := make(map[string]bool)
|
|
// Map to track deletion events by kind, pubkey, and d-tag (for
|
|
// parameterized replaceable events). We store the newest delete timestamp per d-tag.
|
|
deletionsByKindPubkeyDTag := make(map[string]map[string]int64)
|
|
// Map to track specific event IDs that have been deleted
|
|
deletedEventIds := make(map[string]bool)
|
|
// Query for deletion events separately if we have authors in the filter
|
|
if f.Authors != nil && f.Authors.Len() > 0 {
|
|
// Create a filter for deletion events with the same authors
|
|
deletionFilter := &filter.F{
|
|
Kinds: kind.NewS(kind.New(5)), // Kind 5 is deletion
|
|
Authors: f.Authors,
|
|
}
|
|
|
|
var deletionIdPkTs []*store.IdPkTs
|
|
if deletionIdPkTs, err = d.QueryForIds(
|
|
c, deletionFilter,
|
|
); chk.E(err) {
|
|
return
|
|
}
|
|
|
|
// Add deletion events to the list of events to process
|
|
idPkTs = append(idPkTs, deletionIdPkTs...)
|
|
}
|
|
// First pass: collect all deletion events
|
|
for _, idpk := range idPkTs {
|
|
var ev *event.E
|
|
ser := new(types.Uint40)
|
|
if err = ser.Set(idpk.Ser); chk.E(err) {
|
|
continue
|
|
}
|
|
if ev, err = d.FetchEventBySerial(ser); err != nil {
|
|
continue
|
|
}
|
|
// check for an expiration tag and delete after returning the result
|
|
if CheckExpiration(ev) {
|
|
expDeletes = append(expDeletes, ser)
|
|
expEvs = append(expEvs, ev)
|
|
continue
|
|
}
|
|
// Process deletion events to build our deletion maps
|
|
if ev.Kind == kind.Deletion.K {
|
|
// Check for 'e' tags that directly reference event IDs
|
|
eTags := ev.Tags.GetAll([]byte("e"))
|
|
for _, eTag := range eTags {
|
|
if eTag.Len() < 2 {
|
|
continue
|
|
}
|
|
// We don't need to do anything with direct event ID
|
|
// references as we will filter those out in the second pass
|
|
}
|
|
// Check for 'a' tags that reference parameterized replaceable
|
|
// events
|
|
aTags := ev.Tags.GetAll([]byte("a"))
|
|
for _, aTag := range aTags {
|
|
if aTag.Len() < 2 {
|
|
continue
|
|
}
|
|
// Parse the 'a' tag value: kind:pubkey:d-tag
|
|
split := bytes.Split(aTag.Value(), []byte{':'})
|
|
if len(split) != 3 {
|
|
continue
|
|
}
|
|
// Parse the kind
|
|
kindStr := string(split[0])
|
|
kindInt, err := strconv.Atoi(kindStr)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
kk := kind.New(uint16(kindInt))
|
|
// Only process parameterized replaceable events
|
|
if !kind.IsParameterizedReplaceable(kk.K) {
|
|
continue
|
|
}
|
|
// Parse the pubkey
|
|
var pk []byte
|
|
if pk, err = hex.DecAppend(nil, split[1]); err != nil {
|
|
continue
|
|
}
|
|
// Only allow users to delete their own events
|
|
if !utils.FastEqual(pk, ev.Pubkey) {
|
|
continue
|
|
}
|
|
// Create the key for the deletion map using hex
|
|
// representation of pubkey
|
|
key := hex.Enc(pk) + ":" + strconv.Itoa(int(kk.K))
|
|
// Initialize the inner map if it doesn't exist
|
|
if _, exists := deletionsByKindPubkeyDTag[key]; !exists {
|
|
deletionsByKindPubkeyDTag[key] = make(map[string]int64)
|
|
}
|
|
// Record the newest delete timestamp for this d-tag
|
|
dValue := string(split[2])
|
|
if ts, ok := deletionsByKindPubkeyDTag[key][dValue]; !ok || ev.CreatedAt > ts {
|
|
deletionsByKindPubkeyDTag[key][dValue] = ev.CreatedAt
|
|
}
|
|
// Debug logging
|
|
}
|
|
// For replaceable events, we need to check if there are any
|
|
// e-tags that reference events with the same kind and pubkey
|
|
for _, eTag := range eTags {
|
|
if eTag.Len() < 2 {
|
|
continue
|
|
}
|
|
// Get the event ID from the e-tag
|
|
evId := make([]byte, sha256.Size)
|
|
if _, err = hex.DecBytes(evId, eTag.Value()); err != nil {
|
|
continue
|
|
}
|
|
// Query for the event
|
|
var targetEvs event.S
|
|
targetEvs, err = d.QueryEvents(
|
|
c, &filter.F{Ids: tag.NewFromBytesSlice(evId)},
|
|
)
|
|
if err != nil || len(targetEvs) == 0 {
|
|
continue
|
|
}
|
|
targetEv := targetEvs[0]
|
|
// Only allow users to delete their own events
|
|
if !utils.FastEqual(targetEv.Pubkey, ev.Pubkey) {
|
|
continue
|
|
}
|
|
// Mark the specific event ID as deleted
|
|
deletedEventIds[hex.Enc(targetEv.ID)] = true
|
|
// If the event is replaceable, mark it as deleted, but only
|
|
// for events older than this one
|
|
if kind.IsReplaceable(targetEv.Kind) {
|
|
key := hex.Enc(targetEv.Pubkey) + ":" + strconv.Itoa(int(targetEv.Kind))
|
|
// We will still use deletionsByKindPubkey, but we'll
|
|
// check timestamps in the second pass
|
|
deletionsByKindPubkey[key] = true
|
|
} else if kind.IsParameterizedReplaceable(targetEv.Kind) {
|
|
// For parameterized replaceable events, we need to
|
|
// consider the 'd' tag
|
|
key := hex.Enc(targetEv.Pubkey) + ":" + strconv.Itoa(int(targetEv.Kind))
|
|
|
|
// Get the 'd' tag value
|
|
dTag := targetEv.Tags.GetFirst([]byte("d"))
|
|
var dValue string
|
|
if dTag != nil && dTag.Len() > 1 {
|
|
dValue = string(dTag.Value())
|
|
} else {
|
|
// If no 'd' tag, use empty string
|
|
dValue = ""
|
|
}
|
|
// Initialize the inner map if it doesn't exist
|
|
if _, exists := deletionsByKindPubkeyDTag[key]; !exists {
|
|
deletionsByKindPubkeyDTag[key] = make(map[string]int64)
|
|
}
|
|
// Record the newest delete timestamp for this d-tag
|
|
if ts, ok := deletionsByKindPubkeyDTag[key][dValue]; !ok || ev.CreatedAt > ts {
|
|
deletionsByKindPubkeyDTag[key][dValue] = ev.CreatedAt
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Second pass: process all events, filtering out deleted ones
|
|
for _, idpk := range idPkTs {
|
|
var ev *event.E
|
|
ser := new(types.Uint40)
|
|
if err = ser.Set(idpk.Ser); chk.E(err) {
|
|
continue
|
|
}
|
|
if ev, err = d.FetchEventBySerial(ser); err != nil {
|
|
continue
|
|
}
|
|
// Add logging for tag filter debugging
|
|
if f.Tags != nil && f.Tags.Len() > 0 {
|
|
// var eventTags []string
|
|
// if ev.Tags != nil && ev.Tags.Len() > 0 {
|
|
// for _, t := range *ev.Tags {
|
|
// if t.Len() >= 2 {
|
|
// eventTags = append(
|
|
// eventTags,
|
|
// string(t.Key())+"="+string(t.Value()),
|
|
// )
|
|
// }
|
|
// }
|
|
// }
|
|
// log.T.F(
|
|
// "QueryEvents: processing event ID=%s kind=%d tags=%v",
|
|
// hex.Enc(ev.ID), ev.Kind, eventTags,
|
|
// )
|
|
// Check if this event matches ALL required tags in the filter
|
|
tagMatches := 0
|
|
for _, filterTag := range *f.Tags {
|
|
if filterTag.Len() >= 2 {
|
|
filterKey := filterTag.Key()
|
|
// Handle filter keys that start with # (remove the prefix for comparison)
|
|
var actualKey []byte
|
|
if len(filterKey) == 2 && filterKey[0] == '#' {
|
|
actualKey = filterKey[1:]
|
|
} else {
|
|
actualKey = filterKey
|
|
}
|
|
// Check if event has this tag key with any of the filter's values
|
|
eventHasTag := false
|
|
if ev.Tags != nil {
|
|
for _, eventTag := range *ev.Tags {
|
|
if eventTag.Len() >= 2 && bytes.Equal(
|
|
eventTag.Key(), actualKey,
|
|
) {
|
|
// Check if the event's tag value matches any of the filter's values
|
|
for _, filterValue := range filterTag.T[1:] {
|
|
if bytes.Equal(
|
|
eventTag.Value(), filterValue,
|
|
) {
|
|
eventHasTag = true
|
|
break
|
|
}
|
|
}
|
|
if eventHasTag {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if eventHasTag {
|
|
tagMatches++
|
|
}
|
|
// log.T.F(
|
|
// "QueryEvents: tag filter %s (actual key: %s) matches: %v (total matches: %d/%d)",
|
|
// string(filterKey), string(actualKey), eventHasTag,
|
|
// tagMatches, f.Tags.Len(),
|
|
// )
|
|
}
|
|
}
|
|
|
|
// If not all tags match, skip this event
|
|
if tagMatches < f.Tags.Len() {
|
|
// log.T.F(
|
|
// "QueryEvents: event ID=%s SKIPPED - only matches %d/%d required tags",
|
|
// hex.Enc(ev.ID), tagMatches, f.Tags.Len(),
|
|
// )
|
|
continue
|
|
}
|
|
// log.T.F(
|
|
// "QueryEvents: event ID=%s PASSES all tag filters",
|
|
// hex.Enc(ev.ID),
|
|
// )
|
|
}
|
|
|
|
// Skip events with kind 5 (Deletion)
|
|
if ev.Kind == kind.Deletion.K {
|
|
continue
|
|
}
|
|
// Check if this event's ID is in the filter
|
|
isIdInFilter := false
|
|
if f.Ids != nil && f.Ids.Len() > 0 {
|
|
for i := 0; i < f.Ids.Len(); i++ {
|
|
if utils.FastEqual(ev.ID, (*f.Ids).T[i]) {
|
|
isIdInFilter = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
// Check if this specific event has been deleted
|
|
eventIdHex := hex.Enc(ev.ID)
|
|
if deletedEventIds[eventIdHex] && !isIdInFilter {
|
|
// Skip this event if it has been specifically deleted and is
|
|
// not in the filter
|
|
continue
|
|
}
|
|
if kind.IsReplaceable(ev.Kind) {
|
|
// For replaceable events, we only keep the latest version for
|
|
// each pubkey and kind, and only if it hasn't been deleted
|
|
key := hex.Enc(ev.Pubkey) + ":" + strconv.Itoa(int(ev.Kind))
|
|
// For replaceable events, we need to be more careful with
|
|
// deletion Only skip this event if it has been deleted by
|
|
// kind/pubkey and is not in the filter AND there isn't a newer
|
|
// event with the same kind/pubkey
|
|
if deletionsByKindPubkey[key] && !isIdInFilter {
|
|
// Check if there's a newer event with the same kind/pubkey
|
|
// that hasn't been specifically deleted
|
|
existing, exists := replaceableEvents[key]
|
|
if !exists || ev.CreatedAt > existing.CreatedAt {
|
|
// This is the newest event so far, keep it
|
|
replaceableEvents[key] = ev
|
|
} else {
|
|
// There's a newer event, skip this one
|
|
continue
|
|
}
|
|
} else {
|
|
// Normal replaceable event handling
|
|
existing, exists := replaceableEvents[key]
|
|
if !exists || ev.CreatedAt > existing.CreatedAt {
|
|
replaceableEvents[key] = ev
|
|
}
|
|
}
|
|
} else if kind.IsParameterizedReplaceable(ev.Kind) {
|
|
// For parameterized replaceable events, we need to consider the
|
|
// 'd' tag
|
|
key := hex.Enc(ev.Pubkey) + ":" + strconv.Itoa(int(ev.Kind))
|
|
|
|
// Get the 'd' tag value
|
|
dTag := ev.Tags.GetFirst([]byte("d"))
|
|
var dValue string
|
|
if dTag != nil && dTag.Len() > 1 {
|
|
dValue = string(dTag.Value())
|
|
} else {
|
|
// If no 'd' tag, use empty string
|
|
dValue = ""
|
|
}
|
|
|
|
// Check if this event has been deleted via an a-tag
|
|
if deletionMap, exists := deletionsByKindPubkeyDTag[key]; exists {
|
|
// If there is a deletion timestamp and this event is older than the deletion,
|
|
// and this event is not specifically requested by ID, skip it
|
|
if delTs, ok := deletionMap[dValue]; ok && ev.CreatedAt < delTs && !isIdInFilter {
|
|
log.T.F("Debug: Event deleted by a-tag (older than delete) - skipping")
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Initialize the inner map if it doesn't exist
|
|
if _, exists := paramReplaceableEvents[key]; !exists {
|
|
paramReplaceableEvents[key] = make(map[string]*event.E)
|
|
}
|
|
|
|
// Check if we already have an event with this 'd' tag value
|
|
existing, exists := paramReplaceableEvents[key][dValue]
|
|
// Only keep the newer event, regardless of processing order
|
|
if !exists {
|
|
// No existing event, add this one
|
|
paramReplaceableEvents[key][dValue] = ev
|
|
} else if ev.CreatedAt > existing.CreatedAt {
|
|
// This event is newer than the existing one, replace it
|
|
paramReplaceableEvents[key][dValue] = ev
|
|
}
|
|
// If this event is older than the existing one, ignore it
|
|
} else {
|
|
// Regular events
|
|
regularEvents = append(regularEvents, ev)
|
|
}
|
|
}
|
|
// Add all the latest replaceable events to the result
|
|
for _, ev := range replaceableEvents {
|
|
evs = append(evs, ev)
|
|
}
|
|
|
|
// Add all the latest parameterized replaceable events to the result
|
|
for _, innerMap := range paramReplaceableEvents {
|
|
for _, ev := range innerMap {
|
|
evs = append(evs, ev)
|
|
}
|
|
}
|
|
// Add all regular events to the result
|
|
evs = append(evs, regularEvents...)
|
|
// Sort all events by timestamp (newest first)
|
|
sort.Slice(
|
|
evs, func(i, j int) bool {
|
|
return evs[i].CreatedAt > evs[j].CreatedAt
|
|
},
|
|
)
|
|
// delete the expired events in a background thread
|
|
go func() {
|
|
for i, ser := range expDeletes {
|
|
if err = d.DeleteEventBySerial(c, ser, expEvs[i]); chk.E(err) {
|
|
continue
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
return
|
|
}
|