package database import ( "bytes" "context" "sort" "strconv" "strings" "time" "lol.mleku.dev/chk" "lol.mleku.dev/log" "next.orly.dev/pkg/crypto/sha256" "next.orly.dev/pkg/database/indexes/types" "next.orly.dev/pkg/encoders/event" "next.orly.dev/pkg/encoders/filter" "next.orly.dev/pkg/encoders/hex" "next.orly.dev/pkg/encoders/ints" "next.orly.dev/pkg/encoders/kind" "next.orly.dev/pkg/encoders/tag" "next.orly.dev/pkg/interfaces/store" "next.orly.dev/pkg/utils" ) func CheckExpiration(ev *event.E) (expired bool) { var err error expTag := ev.Tags.GetFirst([]byte("expiration")) if expTag != nil { expTS := ints.New(0) if _, err = expTS.Unmarshal(expTag.Value()); !chk.E(err) { if int64(expTS.N) < time.Now().Unix() { return true } } } return } func (d *D) QueryEvents(c context.Context, f *filter.F) ( evs event.S, err error, ) { // if there is Ids in the query, this overrides anything else var expDeletes types.Uint40s var expEvs event.S if f.Ids != nil && f.Ids.Len() > 0 { // for _, id := range f.Ids.T { // log.T.F("QueryEvents: looking for ID=%s", hex.Enc(id)) // } // log.T.F("QueryEvents: ids path, count=%d", f.Ids.Len()) for _, idx := range f.Ids.T { // log.T.F("QueryEvents: lookup id=%s", hex.Enc(idx)) // we know there is only Ids in this, so run the ID query and fetch. var ser *types.Uint40 var idErr error if ser, idErr = d.GetSerialById(idx); idErr != nil { // Check if this is a "not found" error which is expected for IDs we don't have if strings.Contains(idErr.Error(), "id not found in database") { // log.T.F( // "QueryEvents: ID not found in database: %s", // hex.Enc(idx), // ) } else { // Log unexpected errors but continue processing other IDs // log.E.F( // "QueryEvents: error looking up id=%s err=%v", // hex.Enc(idx), idErr, // ) } continue } // Check if the serial is nil, which indicates the ID wasn't found if ser == nil { // log.T.F("QueryEvents: Serial is nil for ID: %s", hex.Enc(idx)) continue } // fetch the events var ev *event.E if ev, err = d.FetchEventBySerial(ser); err != nil { // log.T.F( // "QueryEvents: fetch by serial failed for id=%s ser=%v err=%v", // hex.Enc(idx), ser, err, // ) continue } // log.T.F( // "QueryEvents: found id=%s kind=%d created_at=%d", // hex.Enc(ev.ID), ev.Kind, ev.CreatedAt, // ) // check for an expiration tag and delete after returning the result if CheckExpiration(ev) { log.T.F( "QueryEvents: id=%s filtered out due to expiration", hex.Enc(ev.ID), ) expDeletes = append(expDeletes, ser) expEvs = append(expEvs, ev) continue } // skip events that have been deleted by a proper deletion event if derr := d.CheckForDeleted(ev, nil); derr != nil { // log.T.F( // "QueryEvents: id=%s filtered out due to deletion: %v", // hex.Enc(ev.ID), derr, // ) continue } // log.T.F( // "QueryEvents: id=%s SUCCESSFULLY FOUND, adding to results", // hex.Enc(ev.ID), // ) evs = append(evs, ev) } // sort the events by timestamp sort.Slice( evs, func(i, j int) bool { return evs[i].CreatedAt > evs[j].CreatedAt }, ) } else { // non-IDs path var idPkTs []*store.IdPkTs // if f.Authors != nil && f.Authors.Len() > 0 && f.Kinds != nil && f.Kinds.Len() > 0 { // log.T.F("QueryEvents: authors+kinds path, authors=%d kinds=%d", f.Authors.Len(), f.Kinds.Len()) // } if idPkTs, err = d.QueryForIds(c, f); chk.E(err) { return } // log.T.F("QueryEvents: QueryForIds returned %d candidates", len(idPkTs)) // Create a map to store the latest version of replaceable events replaceableEvents := make(map[string]*event.E) // Create a map to store the latest version of parameterized replaceable // events paramReplaceableEvents := make(map[string]map[string]*event.E) // Regular events that are not replaceable var regularEvents event.S // Map to track deletion events by kind and pubkey (for replaceable // events) deletionsByKindPubkey := make(map[string]bool) // Map to track deletion events by kind, pubkey, and d-tag (for // parameterized replaceable events). We store the newest delete timestamp per d-tag. deletionsByKindPubkeyDTag := make(map[string]map[string]int64) // Map to track specific event IDs that have been deleted deletedEventIds := make(map[string]bool) // Query for deletion events separately if we have authors in the filter if f.Authors != nil && f.Authors.Len() > 0 { // Create a filter for deletion events with the same authors deletionFilter := &filter.F{ Kinds: kind.NewS(kind.New(5)), // Kind 5 is deletion Authors: f.Authors, } var deletionIdPkTs []*store.IdPkTs if deletionIdPkTs, err = d.QueryForIds( c, deletionFilter, ); chk.E(err) { return } // Add deletion events to the list of events to process idPkTs = append(idPkTs, deletionIdPkTs...) } // First pass: collect all deletion events for _, idpk := range idPkTs { var ev *event.E ser := new(types.Uint40) if err = ser.Set(idpk.Ser); chk.E(err) { continue } if ev, err = d.FetchEventBySerial(ser); err != nil { continue } // check for an expiration tag and delete after returning the result if CheckExpiration(ev) { expDeletes = append(expDeletes, ser) expEvs = append(expEvs, ev) continue } // Process deletion events to build our deletion maps if ev.Kind == kind.Deletion.K { // Check for 'e' tags that directly reference event IDs eTags := ev.Tags.GetAll([]byte("e")) for _, eTag := range eTags { if eTag.Len() < 2 { continue } // We don't need to do anything with direct event ID // references as we will filter those out in the second pass } // Check for 'a' tags that reference parameterized replaceable // events aTags := ev.Tags.GetAll([]byte("a")) for _, aTag := range aTags { if aTag.Len() < 2 { continue } // Parse the 'a' tag value: kind:pubkey:d-tag split := bytes.Split(aTag.Value(), []byte{':'}) if len(split) != 3 { continue } // Parse the kind kindStr := string(split[0]) kindInt, err := strconv.Atoi(kindStr) if err != nil { continue } kk := kind.New(uint16(kindInt)) // Only process parameterized replaceable events if !kind.IsParameterizedReplaceable(kk.K) { continue } // Parse the pubkey var pk []byte if pk, err = hex.DecAppend(nil, split[1]); err != nil { continue } // Only allow users to delete their own events if !utils.FastEqual(pk, ev.Pubkey) { continue } // Create the key for the deletion map using hex // representation of pubkey key := hex.Enc(pk) + ":" + strconv.Itoa(int(kk.K)) // Initialize the inner map if it doesn't exist if _, exists := deletionsByKindPubkeyDTag[key]; !exists { deletionsByKindPubkeyDTag[key] = make(map[string]int64) } // Record the newest delete timestamp for this d-tag dValue := string(split[2]) if ts, ok := deletionsByKindPubkeyDTag[key][dValue]; !ok || ev.CreatedAt > ts { deletionsByKindPubkeyDTag[key][dValue] = ev.CreatedAt } // Debug logging } // For replaceable events, we need to check if there are any // e-tags that reference events with the same kind and pubkey for _, eTag := range eTags { if eTag.Len() < 2 { continue } // Get the event ID from the e-tag evId := make([]byte, sha256.Size) if _, err = hex.DecBytes(evId, eTag.Value()); err != nil { continue } // Query for the event var targetEvs event.S targetEvs, err = d.QueryEvents( c, &filter.F{Ids: tag.NewFromBytesSlice(evId)}, ) if err != nil || len(targetEvs) == 0 { continue } targetEv := targetEvs[0] // Only allow users to delete their own events if !utils.FastEqual(targetEv.Pubkey, ev.Pubkey) { continue } // Mark the specific event ID as deleted deletedEventIds[hex.Enc(targetEv.ID)] = true // If the event is replaceable, mark it as deleted, but only // for events older than this one if kind.IsReplaceable(targetEv.Kind) { key := hex.Enc(targetEv.Pubkey) + ":" + strconv.Itoa(int(targetEv.Kind)) // We will still use deletionsByKindPubkey, but we'll // check timestamps in the second pass deletionsByKindPubkey[key] = true } else if kind.IsParameterizedReplaceable(targetEv.Kind) { // For parameterized replaceable events, we need to // consider the 'd' tag key := hex.Enc(targetEv.Pubkey) + ":" + strconv.Itoa(int(targetEv.Kind)) // Get the 'd' tag value dTag := targetEv.Tags.GetFirst([]byte("d")) var dValue string if dTag != nil && dTag.Len() > 1 { dValue = string(dTag.Value()) } else { // If no 'd' tag, use empty string dValue = "" } // Initialize the inner map if it doesn't exist if _, exists := deletionsByKindPubkeyDTag[key]; !exists { deletionsByKindPubkeyDTag[key] = make(map[string]int64) } // Record the newest delete timestamp for this d-tag if ts, ok := deletionsByKindPubkeyDTag[key][dValue]; !ok || ev.CreatedAt > ts { deletionsByKindPubkeyDTag[key][dValue] = ev.CreatedAt } } } } } // Second pass: process all events, filtering out deleted ones for _, idpk := range idPkTs { var ev *event.E ser := new(types.Uint40) if err = ser.Set(idpk.Ser); chk.E(err) { continue } if ev, err = d.FetchEventBySerial(ser); err != nil { continue } // Add logging for tag filter debugging if f.Tags != nil && f.Tags.Len() > 0 { // var eventTags []string // if ev.Tags != nil && ev.Tags.Len() > 0 { // for _, t := range *ev.Tags { // if t.Len() >= 2 { // eventTags = append( // eventTags, // string(t.Key())+"="+string(t.Value()), // ) // } // } // } // log.T.F( // "QueryEvents: processing event ID=%s kind=%d tags=%v", // hex.Enc(ev.ID), ev.Kind, eventTags, // ) // Check if this event matches ALL required tags in the filter tagMatches := 0 for _, filterTag := range *f.Tags { if filterTag.Len() >= 2 { filterKey := filterTag.Key() // Handle filter keys that start with # (remove the prefix for comparison) var actualKey []byte if len(filterKey) == 2 && filterKey[0] == '#' { actualKey = filterKey[1:] } else { actualKey = filterKey } // Check if event has this tag key with any of the filter's values eventHasTag := false if ev.Tags != nil { for _, eventTag := range *ev.Tags { if eventTag.Len() >= 2 && bytes.Equal( eventTag.Key(), actualKey, ) { // Check if the event's tag value matches any of the filter's values for _, filterValue := range filterTag.T[1:] { if bytes.Equal( eventTag.Value(), filterValue, ) { eventHasTag = true break } } if eventHasTag { break } } } } if eventHasTag { tagMatches++ } // log.T.F( // "QueryEvents: tag filter %s (actual key: %s) matches: %v (total matches: %d/%d)", // string(filterKey), string(actualKey), eventHasTag, // tagMatches, f.Tags.Len(), // ) } } // If not all tags match, skip this event if tagMatches < f.Tags.Len() { // log.T.F( // "QueryEvents: event ID=%s SKIPPED - only matches %d/%d required tags", // hex.Enc(ev.ID), tagMatches, f.Tags.Len(), // ) continue } // log.T.F( // "QueryEvents: event ID=%s PASSES all tag filters", // hex.Enc(ev.ID), // ) } // Skip events with kind 5 (Deletion) if ev.Kind == kind.Deletion.K { continue } // Check if this event's ID is in the filter isIdInFilter := false if f.Ids != nil && f.Ids.Len() > 0 { for i := 0; i < f.Ids.Len(); i++ { if utils.FastEqual(ev.ID, (*f.Ids).T[i]) { isIdInFilter = true break } } } // Check if this specific event has been deleted eventIdHex := hex.Enc(ev.ID) if deletedEventIds[eventIdHex] && !isIdInFilter { // Skip this event if it has been specifically deleted and is // not in the filter continue } if kind.IsReplaceable(ev.Kind) { // For replaceable events, we only keep the latest version for // each pubkey and kind, and only if it hasn't been deleted key := hex.Enc(ev.Pubkey) + ":" + strconv.Itoa(int(ev.Kind)) // For replaceable events, we need to be more careful with // deletion Only skip this event if it has been deleted by // kind/pubkey and is not in the filter AND there isn't a newer // event with the same kind/pubkey if deletionsByKindPubkey[key] && !isIdInFilter { // Check if there's a newer event with the same kind/pubkey // that hasn't been specifically deleted existing, exists := replaceableEvents[key] if !exists || ev.CreatedAt > existing.CreatedAt { // This is the newest event so far, keep it replaceableEvents[key] = ev } else { // There's a newer event, skip this one continue } } else { // Normal replaceable event handling existing, exists := replaceableEvents[key] if !exists || ev.CreatedAt > existing.CreatedAt { replaceableEvents[key] = ev } } } else if kind.IsParameterizedReplaceable(ev.Kind) { // For parameterized replaceable events, we need to consider the // 'd' tag key := hex.Enc(ev.Pubkey) + ":" + strconv.Itoa(int(ev.Kind)) // Get the 'd' tag value dTag := ev.Tags.GetFirst([]byte("d")) var dValue string if dTag != nil && dTag.Len() > 1 { dValue = string(dTag.Value()) } else { // If no 'd' tag, use empty string dValue = "" } // Check if this event has been deleted via an a-tag if deletionMap, exists := deletionsByKindPubkeyDTag[key]; exists { // If there is a deletion timestamp and this event is older than the deletion, // and this event is not specifically requested by ID, skip it if delTs, ok := deletionMap[dValue]; ok && ev.CreatedAt < delTs && !isIdInFilter { log.T.F("Debug: Event deleted by a-tag (older than delete) - skipping") continue } } // Initialize the inner map if it doesn't exist if _, exists := paramReplaceableEvents[key]; !exists { paramReplaceableEvents[key] = make(map[string]*event.E) } // Check if we already have an event with this 'd' tag value existing, exists := paramReplaceableEvents[key][dValue] // Only keep the newer event, regardless of processing order if !exists { // No existing event, add this one paramReplaceableEvents[key][dValue] = ev } else if ev.CreatedAt > existing.CreatedAt { // This event is newer than the existing one, replace it paramReplaceableEvents[key][dValue] = ev } // If this event is older than the existing one, ignore it } else { // Regular events regularEvents = append(regularEvents, ev) } } // Add all the latest replaceable events to the result for _, ev := range replaceableEvents { evs = append(evs, ev) } // Add all the latest parameterized replaceable events to the result for _, innerMap := range paramReplaceableEvents { for _, ev := range innerMap { evs = append(evs, ev) } } // Add all regular events to the result evs = append(evs, regularEvents...) // Sort all events by timestamp (newest first) sort.Slice( evs, func(i, j int) bool { return evs[i].CreatedAt > evs[j].CreatedAt }, ) // delete the expired events in a background thread go func() { for i, ser := range expDeletes { if err = d.DeleteEventBySerial(c, ser, expEvs[i]); chk.E(err) { continue } } }() } return }