Compare commits

..

3 Commits

Author SHA1 Message Date
6b72f1f2b7 Update privileged event filtering to respect ACL mode
Some checks failed
Go / build-and-release (push) Has been cancelled
Privileged events are now filtered based on ACL mode, allowing open access when ACL is "none." Added tests to verify behavior for different ACL modes, ensuring unauthorized and unauthenticated users can only access privileged events when explicitly permitted. Version bumped to v0.34.2.
2025-12-05 10:02:49 +00:00
83c27a52b0 bump v0.34.1
Some checks failed
Go / build-and-release (push) Has been cancelled
2025-12-04 20:10:27 +00:00
1e9c447fe6 Refactor Neo4j tests and improve tag handling in Cypher
Replaces outdated Neo4j test setup with a robust TestMain, shared test database, and utility functions for test data and migrations. Improves Cypher generation for processing e-tags, p-tags, and other tags to ensure compliance with Neo4j syntax. Added integration test script and updated benchmark reports for Badger backend.
2025-12-04 20:09:24 +00:00
31 changed files with 3389 additions and 171 deletions

View File

@@ -25,7 +25,15 @@ func (l *Listener) HandleDelete(env *eventenvelope.Submission) (err error) {
log.I.F("HandleDelete: processing delete event %0x from pubkey %0x", env.E.ID, env.E.Pubkey)
log.I.F("HandleDelete: delete event tags: %d tags", len(*env.E.Tags))
for i, t := range *env.E.Tags {
log.I.F("HandleDelete: tag %d: %s = %s", i, string(t.Key()), string(t.Value()))
// Use ValueHex() for e/p tags to properly display binary-encoded values
key := string(t.Key())
var val string
if key == "e" || key == "p" {
val = string(t.ValueHex()) // Properly converts binary to hex
} else {
val = string(t.Value())
}
log.I.F("HandleDelete: tag %d: %s = %s", i, key, val)
}
// Debug: log admin and owner lists
@@ -142,27 +150,21 @@ func (l *Listener) HandleDelete(env *eventenvelope.Submission) (err error) {
// if e tags are found, delete them if the author is signer, or one of
// the owners is signer
if utils.FastEqual(t.Key(), []byte("e")) {
// First try binary format (optimized storage for e-tags)
var dst []byte
if binVal := t.ValueBinary(); binVal != nil {
dst = binVal
log.I.F("HandleDelete: processing binary e-tag event ID: %0x", dst)
} else {
// Fall back to hex decoding for non-binary values
val := t.Value()
if len(val) == 0 {
log.W.F("HandleDelete: empty e-tag value")
continue
}
log.I.F("HandleDelete: processing e-tag with value: %s", string(val))
if b, e := hex.Dec(string(val)); chk.E(e) {
log.E.F("HandleDelete: failed to decode hex event ID %s: %v", string(val), e)
continue
} else {
dst = b
log.I.F("HandleDelete: decoded event ID: %0x", dst)
}
// Use ValueHex() which properly handles both binary-encoded and hex string formats
hexVal := t.ValueHex()
if len(hexVal) == 0 {
log.W.F("HandleDelete: empty e-tag value")
continue
}
log.I.F("HandleDelete: processing e-tag event ID: %s", string(hexVal))
// Decode hex to binary for filter
dst, e := hex.Dec(string(hexVal))
if chk.E(e) {
log.E.F("HandleDelete: failed to decode event ID %s: %v", string(hexVal), e)
continue
}
f := &filter.F{
Ids: tag.NewFromBytesSlice(dst),
}

View File

@@ -439,10 +439,12 @@ func (l *Listener) HandleReq(msg []byte) (err error) {
// Event has private tag and user is authorized - continue to privileged check
}
// Always filter privileged events based on kind, regardless of ACLMode
// Filter privileged events based on kind when ACL is active
// When ACL is "none", skip privileged filtering to allow open access
// Privileged events should only be sent to users who are authenticated and
// are either the event author or listed in p tags
if kind.IsPrivileged(ev.Kind) && accessLevel != "admin" { // admins can see all events
aclActive := acl.Registry.Active.Load() != "none"
if kind.IsPrivileged(ev.Kind) && aclActive && accessLevel != "admin" { // admins can see all events
log.T.C(
func() string {
return fmt.Sprintf(

View File

@@ -404,6 +404,82 @@ func TestPrivilegedEventEdgeCases(t *testing.T) {
}
}
// TestPrivilegedEventsWithACLNone tests that privileged events are accessible
// to anyone when ACL mode is set to "none" (open relay)
func TestPrivilegedEventsWithACLNone(t *testing.T) {
authorPubkey := []byte("author-pubkey-12345")
recipientPubkey := []byte("recipient-pubkey-67")
unauthorizedPubkey := []byte("unauthorized-pubkey")
// Create a privileged event (encrypted DM)
privilegedEvent := createTestEvent(
"event-id-1",
hex.Enc(authorPubkey),
"private message",
kind.EncryptedDirectMessage.K,
createPTag(hex.Enc(recipientPubkey)),
)
tests := []struct {
name string
authedPubkey []byte
aclMode string
accessLevel string
shouldAllow bool
description string
}{
{
name: "ACL none - unauthorized user can see privileged event",
authedPubkey: unauthorizedPubkey,
aclMode: "none",
accessLevel: "write", // default for ACL=none
shouldAllow: true,
description: "When ACL is 'none', privileged events should be visible to anyone",
},
{
name: "ACL none - unauthenticated user can see privileged event",
authedPubkey: nil,
aclMode: "none",
accessLevel: "write", // default for ACL=none
shouldAllow: true,
description: "When ACL is 'none', even unauthenticated users can see privileged events",
},
{
name: "ACL managed - unauthorized user cannot see privileged event",
authedPubkey: unauthorizedPubkey,
aclMode: "managed",
accessLevel: "write",
shouldAllow: false,
description: "When ACL is 'managed', unauthorized users cannot see privileged events",
},
{
name: "ACL follows - unauthorized user cannot see privileged event",
authedPubkey: unauthorizedPubkey,
aclMode: "follows",
accessLevel: "write",
shouldAllow: false,
description: "When ACL is 'follows', unauthorized users cannot see privileged events",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
events := event.S{privilegedEvent}
filtered := testPrivilegedEventFiltering(events, tt.authedPubkey, tt.aclMode, tt.accessLevel)
if tt.shouldAllow {
if len(filtered) != 1 {
t.Errorf("%s: Expected event to be allowed, but it was filtered out. %s", tt.name, tt.description)
}
} else {
if len(filtered) != 0 {
t.Errorf("%s: Expected event to be filtered out, but it was allowed. %s", tt.name, tt.description)
}
}
})
}
}
func TestPrivilegedEventPolicyIntegration(t *testing.T) {
// Test that the policy system also correctly handles privileged events
// This tests the policy.go implementation

View File

@@ -11,7 +11,12 @@
"Bash(docker compose:*)",
"Bash(./run-benchmark.sh:*)",
"Bash(tee:*)",
"Bash(sudo rm:*)"
"Bash(sudo rm:*)",
"Bash(go mod tidy:*)",
"Bash(docker run --rm -v \"/home/mleku/src/next.orly.dev/cmd/benchmark/data:/data\" --user root alpine sh -c \"rm -rf /data/* /data/.[!.]*\")",
"Bash(head:*)",
"Bash(cat:*)",
"Bash(chmod:*)"
],
"deny": [],
"ask": []

View File

@@ -18,8 +18,11 @@ RUN git clone https://git.nostrdev.com/mleku/next.orly.dev.git . && \
echo "Building benchmark from ORLY version: ${LATEST_TAG}" && \
git checkout "${LATEST_TAG}"
# Download dependencies
RUN go mod download
# Remove local replace directives and update to released version, then download dependencies
RUN sed -i '/^replace .* => \/home/d' go.mod && \
sed -i 's/git.mleku.dev\/mleku\/nostr v1.0.7/git.mleku.dev\/mleku\/nostr v1.0.8/' go.mod && \
go mod tidy && \
go mod download
# Build the benchmark tool with CGO disabled (uses purego for crypto)
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o benchmark ./cmd/benchmark

View File

@@ -19,8 +19,11 @@ RUN git clone https://git.nostrdev.com/mleku/next.orly.dev.git . && \
echo "Building ORLY version: ${LATEST_TAG}" && \
git checkout "${LATEST_TAG}"
# Download dependencies
RUN go mod download
# Remove local replace directives and update to released version, then download dependencies
RUN sed -i '/^replace .* => \/home/d' go.mod && \
sed -i 's/git.mleku.dev\/mleku\/nostr v1.0.7/git.mleku.dev\/mleku\/nostr v1.0.8/' go.mod && \
go mod tidy && \
go mod download
# Build the relay with CGO disabled (uses purego for crypto)
# Include debug symbols for profiling

View File

@@ -0,0 +1,74 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_khatru-badger_8
Events: 50000, Workers: 24, Duration: 1m0s
1764840830987179 migrating to version 1... /build/pkg/database/migrations.go:68
1764840830987255 migrating to version 2... /build/pkg/database/migrations.go:75
1764840830987278 migrating to version 3... /build/pkg/database/migrations.go:82
1764840830987283 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764840830987292 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764840830987305 migrating to version 4... /build/pkg/database/migrations.go:89
1764840830987310 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764840830987336 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764840830987364 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764840830987412 migrating to version 5... /build/pkg/database/migrations.go:96
1764840830987419 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764840830987429 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764840830987435 no events need re-encoding /build/pkg/database/migrations.go:652
1764840830987452 migrating to version 6... /build/pkg/database/migrations.go:103
1764840830987458 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764840830987473 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764840830987479 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 09:33:50 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 3.213866224s
Events/sec: 15557.59
Avg latency: 1.456848ms
P90 latency: 1.953553ms
P95 latency: 2.322455ms
P99 latency: 4.316566ms
Bottom 10% Avg latency: 793.956µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 336.223018ms
Burst completed: 5000 events in 314.023603ms
Burst completed: 5000 events in 296.961158ms
Burst completed: 5000 events in 313.470891ms
Burst completed: 5000 events in 312.977339ms
Burst completed: 5000 events in 304.290846ms
Burst completed: 5000 events in 279.718158ms
Burst completed: 5000 events in 351.360773ms
Burst completed: 5000 events in 413.446584ms
Burst completed: 5000 events in 412.074279ms
Burst test completed: 50000 events in 8.341599033s, errors: 0
Events/sec: 5994.05
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.442820936s
Combined ops/sec: 2045.59
Wiping database between tests...

View File

@@ -0,0 +1,8 @@
RELAY_NAME: khatru-sqlite
RELAY_URL: ws://khatru-sqlite:3334
TEST_TIMESTAMP: 2025-12-04T09:33:45+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -1,78 +0,0 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_next-orly-neo4j_8
Events: 50000, Workers: 24, Duration: 1m0s
1764840427673892 migrating to version 1... /build/pkg/database/migrations.go:68
1764840427674007 migrating to version 2... /build/pkg/database/migrations.go:75
1764840427674031 migrating to version 3... /build/pkg/database/migrations.go:82
1764840427674036 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764840427674056 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764840427674081 migrating to version 4... /build/pkg/database/migrations.go:89
1764840427674087 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764840427674097 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764840427674102 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764840427674116 migrating to version 5... /build/pkg/database/migrations.go:96
1764840427674121 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764840427674128 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764840427674132 no events need re-encoding /build/pkg/database/migrations.go:652
1764840427674146 migrating to version 6... /build/pkg/database/migrations.go:103
1764840427674151 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764840427674168 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764840427674172 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 09:27:07 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 3.004845722s
Events/sec: 16639.79
Avg latency: 1.323689ms
P90 latency: 1.758038ms
P95 latency: 2.077948ms
P99 latency: 3.856256ms
Bottom 10% Avg latency: 730.568µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 283.966934ms
Burst completed: 5000 events in 294.692625ms
Burst completed: 5000 events in 363.280618ms
Burst completed: 5000 events in 340.745621ms
Burst completed: 5000 events in 304.674199ms
Burst completed: 5000 events in 280.09038ms
Burst completed: 5000 events in 266.781378ms
Burst completed: 5000 events in 277.70181ms
Burst completed: 5000 events in 271.658408ms
Burst completed: 5000 events in 309.272288ms
Burst test completed: 50000 events in 8.000384614s, errors: 0
Events/sec: 6249.70
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.409054146s
Combined ops/sec: 2048.42
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...

View File

@@ -0,0 +1,201 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_khatru-badger_8
Events: 50000, Workers: 24, Duration: 1m0s
1764845904475025 migrating to version 1... /build/pkg/database/migrations.go:68
1764845904475112 migrating to version 2... /build/pkg/database/migrations.go:75
1764845904475134 migrating to version 3... /build/pkg/database/migrations.go:82
1764845904475139 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764845904475152 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764845904475166 migrating to version 4... /build/pkg/database/migrations.go:89
1764845904475171 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764845904475182 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764845904475187 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764845904475202 migrating to version 5... /build/pkg/database/migrations.go:96
1764845904475207 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764845904475213 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764845904475218 no events need re-encoding /build/pkg/database/migrations.go:652
1764845904475233 migrating to version 6... /build/pkg/database/migrations.go:103
1764845904475238 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764845904475247 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764845904475252 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 10:58:24 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 4.536980771s
Events/sec: 11020.54
Avg latency: 2.141467ms
P90 latency: 3.415814ms
P95 latency: 4.218151ms
P99 latency: 6.573395ms
Bottom 10% Avg latency: 965.163µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 562.534206ms
Burst completed: 5000 events in 495.672511ms
Burst completed: 5000 events in 403.9333ms
Burst completed: 5000 events in 406.633831ms
Burst completed: 5000 events in 497.747932ms
Burst completed: 5000 events in 375.06022ms
Burst completed: 5000 events in 357.935146ms
Burst completed: 5000 events in 354.7018ms
Burst completed: 5000 events in 363.034284ms
Burst completed: 5000 events in 369.648798ms
Burst test completed: 50000 events in 9.192909424s, errors: 0
Events/sec: 5438.97
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.759007602s
Combined ops/sec: 2019.47
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 279947 queries in 1m0.0101769s
Queries/sec: 4664.99
Avg query latency: 3.577317ms
P95 query latency: 13.542975ms
P99 query latency: 20.687227ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 236582 operations (186582 queries, 50000 writes) in 1m0.004658961s
Operations/sec: 3942.73
Avg latency: 2.272206ms
Avg query latency: 2.486915ms
Avg write latency: 1.470991ms
P95 latency: 6.629071ms
P99 latency: 17.102632ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 4.536980771s
Total Events: 50000
Events/sec: 11020.54
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 233 MB
Avg Latency: 2.141467ms
P90 Latency: 3.415814ms
P95 Latency: 4.218151ms
P99 Latency: 6.573395ms
Bottom 10% Avg Latency: 965.163µs
----------------------------------------
Test: Burst Pattern
Duration: 9.192909424s
Total Events: 50000
Events/sec: 5438.97
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 235 MB
Avg Latency: 1.990208ms
P90 Latency: 3.107457ms
P95 Latency: 3.856432ms
P99 Latency: 6.336835ms
Bottom 10% Avg Latency: 900.221µs
----------------------------------------
Test: Mixed Read/Write
Duration: 24.759007602s
Total Events: 50000
Events/sec: 2019.47
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 174 MB
Avg Latency: 450.921µs
P90 Latency: 937.184µs
P95 Latency: 1.10841ms
P99 Latency: 1.666212ms
Bottom 10% Avg Latency: 1.296193ms
----------------------------------------
Test: Query Performance
Duration: 1m0.0101769s
Total Events: 279947
Events/sec: 4664.99
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 142 MB
Avg Latency: 3.577317ms
P90 Latency: 10.560196ms
P95 Latency: 13.542975ms
P99 Latency: 20.687227ms
Bottom 10% Avg Latency: 14.957911ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.004658961s
Total Events: 236582
Events/sec: 3942.73
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 161 MB
Avg Latency: 2.272206ms
P90 Latency: 4.975152ms
P95 Latency: 6.629071ms
P99 Latency: 17.102632ms
Bottom 10% Avg Latency: 8.89611ms
----------------------------------------
Report saved to: /tmp/benchmark_khatru-badger_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_khatru-badger_8/benchmark_report.adoc
RELAY_NAME: khatru-badger
RELAY_URL: ws://khatru-badger:3334
TEST_TIMESTAMP: 2025-12-04T11:01:44+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -0,0 +1,201 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_khatru-sqlite_8
Events: 50000, Workers: 24, Duration: 1m0s
1764845699509026 migrating to version 1... /build/pkg/database/migrations.go:68
1764845699509106 migrating to version 2... /build/pkg/database/migrations.go:75
1764845699509128 migrating to version 3... /build/pkg/database/migrations.go:82
1764845699509133 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764845699509146 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764845699509159 migrating to version 4... /build/pkg/database/migrations.go:89
1764845699509164 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764845699509172 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764845699509178 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764845699509192 migrating to version 5... /build/pkg/database/migrations.go:96
1764845699509197 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764845699509206 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764845699509211 no events need re-encoding /build/pkg/database/migrations.go:652
1764845699509224 migrating to version 6... /build/pkg/database/migrations.go:103
1764845699509228 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764845699509238 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764845699509242 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 10:54:59 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 4.109596583s
Events/sec: 12166.64
Avg latency: 1.93573ms
P90 latency: 2.871977ms
P95 latency: 3.44036ms
P99 latency: 5.475515ms
Bottom 10% Avg latency: 961.636µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 515.356224ms
Burst completed: 5000 events in 399.9581ms
Burst completed: 5000 events in 459.416277ms
Burst completed: 5000 events in 428.20652ms
Burst completed: 5000 events in 747.547021ms
Burst completed: 5000 events in 647.984214ms
Burst completed: 5000 events in 488.90592ms
Burst completed: 5000 events in 377.505306ms
Burst completed: 5000 events in 465.109125ms
Burst completed: 5000 events in 429.364917ms
Burst test completed: 50000 events in 9.965909051s, errors: 0
Events/sec: 5017.10
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.612452482s
Combined ops/sec: 2031.49
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 302291 queries in 1m0.005394665s
Queries/sec: 5037.73
Avg query latency: 3.277291ms
P95 query latency: 12.307232ms
P99 query latency: 18.488169ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 243436 operations (193436 queries, 50000 writes) in 1m0.00468811s
Operations/sec: 4056.95
Avg latency: 2.220391ms
Avg query latency: 2.393422ms
Avg write latency: 1.550983ms
P95 latency: 6.295105ms
P99 latency: 16.788623ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 4.109596583s
Total Events: 50000
Events/sec: 12166.64
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 243 MB
Avg Latency: 1.93573ms
P90 Latency: 2.871977ms
P95 Latency: 3.44036ms
P99 Latency: 5.475515ms
Bottom 10% Avg Latency: 961.636µs
----------------------------------------
Test: Burst Pattern
Duration: 9.965909051s
Total Events: 50000
Events/sec: 5017.10
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 257 MB
Avg Latency: 2.375602ms
P90 Latency: 3.854368ms
P95 Latency: 5.019226ms
P99 Latency: 8.287248ms
Bottom 10% Avg Latency: 1.013228ms
----------------------------------------
Test: Mixed Read/Write
Duration: 24.612452482s
Total Events: 50000
Events/sec: 2031.49
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 190 MB
Avg Latency: 432.265µs
P90 Latency: 913.499µs
P95 Latency: 1.051763ms
P99 Latency: 1.395767ms
Bottom 10% Avg Latency: 1.160261ms
----------------------------------------
Test: Query Performance
Duration: 1m0.005394665s
Total Events: 302291
Events/sec: 5037.73
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 234 MB
Avg Latency: 3.277291ms
P90 Latency: 9.787032ms
P95 Latency: 12.307232ms
P99 Latency: 18.488169ms
Bottom 10% Avg Latency: 13.509646ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.00468811s
Total Events: 243436
Events/sec: 4056.95
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 148 MB
Avg Latency: 2.220391ms
P90 Latency: 4.746928ms
P95 Latency: 6.295105ms
P99 Latency: 16.788623ms
Bottom 10% Avg Latency: 8.681502ms
----------------------------------------
Report saved to: /tmp/benchmark_khatru-sqlite_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_khatru-sqlite_8/benchmark_report.adoc
RELAY_NAME: khatru-sqlite
RELAY_URL: ws://khatru-sqlite:3334
TEST_TIMESTAMP: 2025-12-04T10:58:19+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -0,0 +1,43 @@
Starting Network Graph Traversal Benchmark
Relay URL: ws://next-orly-badger:8080
Workers: 24
Pubkeys: 100000, Follows per pubkey: 1-1000
╔════════════════════════════════════════════════════════╗
║ NETWORK GRAPH TRAVERSAL BENCHMARK (100k Pubkeys) ║
║ Relay: ws://next-orly-badger:8080 ║
╚════════════════════════════════════════════════════════╝
Generating 100000 deterministic pubkeys...
2025/12/04 13:19:05 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Generated 10000/100000 pubkeys...
Generated 20000/100000 pubkeys...
Generated 30000/100000 pubkeys...
Generated 40000/100000 pubkeys...
Generated 50000/100000 pubkeys...
Generated 60000/100000 pubkeys...
Generated 70000/100000 pubkeys...
Generated 80000/100000 pubkeys...
Generated 90000/100000 pubkeys...
Generated 100000/100000 pubkeys...
Generated 100000 pubkeys in 2.473794335s
Generating follow graph (1-1000 follows per pubkey)...
Generated follow lists for 10000/100000 pubkeys...
Generated follow lists for 20000/100000 pubkeys...
Generated follow lists for 30000/100000 pubkeys...
Generated follow lists for 40000/100000 pubkeys...
Generated follow lists for 50000/100000 pubkeys...
Generated follow lists for 60000/100000 pubkeys...
Generated follow lists for 70000/100000 pubkeys...
Generated follow lists for 80000/100000 pubkeys...
Generated follow lists for 90000/100000 pubkeys...
Generated follow lists for 100000/100000 pubkeys...
Generated follow graph in 4.361425602s (avg 500.5 follows/pubkey, total 50048088 follows)
Connecting to relay: ws://next-orly-badger:8080
Connected successfully!
Creating follow list events via WebSocket...
Queued 10000/100000 follow list events...
Queued 20000/100000 follow list events...
Queued 30000/100000 follow list events...
1764854401568817🚨 NOTICE from ws://next-orly-badger:8080: 'EVENT processing failed' /go/pkg/mod/git.mleku.dev/mleku/nostr@v1.0.8/ws/client.go:326
1764854402773843🚨 failed to write message: %!w(*net.OpError=&{write tcp 0xc0001b0f30 0xc0001b0f60 {}}) /go/pkg/mod/git.mleku.dev/mleku/nostr@v1.0.8/ws/connection.go:63

View File

@@ -0,0 +1,201 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_next-orly-badger_8
Events: 50000, Workers: 24, Duration: 1m0s
1764845290757888 migrating to version 1... /build/pkg/database/migrations.go:68
1764845290758084 migrating to version 2... /build/pkg/database/migrations.go:75
1764845290758119 migrating to version 3... /build/pkg/database/migrations.go:82
1764845290758124 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764845290758135 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764845290758150 migrating to version 4... /build/pkg/database/migrations.go:89
1764845290758155 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764845290758167 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764845290758173 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764845290758190 migrating to version 5... /build/pkg/database/migrations.go:96
1764845290758195 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764845290758204 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764845290758210 no events need re-encoding /build/pkg/database/migrations.go:652
1764845290758224 migrating to version 6... /build/pkg/database/migrations.go:103
1764845290758229 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764845290758241 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764845290758247 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 10:48:10 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 4.113585513s
Events/sec: 12154.85
Avg latency: 1.935424ms
P90 latency: 2.908617ms
P95 latency: 3.52541ms
P99 latency: 5.586614ms
Bottom 10% Avg latency: 943.568µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 384.404827ms
Burst completed: 5000 events in 366.066982ms
Burst completed: 5000 events in 413.972961ms
Burst completed: 5000 events in 540.992935ms
Burst completed: 5000 events in 444.488278ms
Burst completed: 5000 events in 342.979185ms
Burst completed: 5000 events in 393.451489ms
Burst completed: 5000 events in 530.328367ms
Burst completed: 5000 events in 483.78923ms
Burst completed: 5000 events in 356.248835ms
Burst test completed: 50000 events in 9.263453685s, errors: 0
Events/sec: 5397.55
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.809227197s
Combined ops/sec: 2015.38
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 256384 queries in 1m0.005966351s
Queries/sec: 4272.64
Avg query latency: 3.92418ms
P95 query latency: 14.841512ms
P99 query latency: 22.768552ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 220975 operations (170975 queries, 50000 writes) in 1m0.003529193s
Operations/sec: 3682.70
Avg latency: 2.572587ms
Avg query latency: 2.803798ms
Avg write latency: 1.781959ms
P95 latency: 7.618974ms
P99 latency: 19.690393ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 4.113585513s
Total Events: 50000
Events/sec: 12154.85
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 127 MB
Avg Latency: 1.935424ms
P90 Latency: 2.908617ms
P95 Latency: 3.52541ms
P99 Latency: 5.586614ms
Bottom 10% Avg Latency: 943.568µs
----------------------------------------
Test: Burst Pattern
Duration: 9.263453685s
Total Events: 50000
Events/sec: 5397.55
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 231 MB
Avg Latency: 2.034536ms
P90 Latency: 3.126682ms
P95 Latency: 3.863975ms
P99 Latency: 6.098539ms
Bottom 10% Avg Latency: 935.662µs
----------------------------------------
Test: Mixed Read/Write
Duration: 24.809227197s
Total Events: 50000
Events/sec: 2015.38
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 184 MB
Avg Latency: 438.529µs
P90 Latency: 917.747µs
P95 Latency: 1.086949ms
P99 Latency: 1.523991ms
Bottom 10% Avg Latency: 1.218802ms
----------------------------------------
Test: Query Performance
Duration: 1m0.005966351s
Total Events: 256384
Events/sec: 4272.64
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 151 MB
Avg Latency: 3.92418ms
P90 Latency: 11.560176ms
P95 Latency: 14.841512ms
P99 Latency: 22.768552ms
Bottom 10% Avg Latency: 16.422096ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.003529193s
Total Events: 220975
Events/sec: 3682.70
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 207 MB
Avg Latency: 2.572587ms
P90 Latency: 5.5629ms
P95 Latency: 7.618974ms
P99 Latency: 19.690393ms
Bottom 10% Avg Latency: 10.306482ms
----------------------------------------
Report saved to: /tmp/benchmark_next-orly-badger_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_next-orly-badger_8/benchmark_report.adoc
RELAY_NAME: next-orly-badger
RELAY_URL: ws://next-orly-badger:8080
TEST_TIMESTAMP: 2025-12-04T10:51:30+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -0,0 +1,201 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_next-orly-neo4j_8
Events: 50000, Workers: 24, Duration: 1m0s
1764845495230040 migrating to version 1... /build/pkg/database/migrations.go:68
1764845495230118 migrating to version 2... /build/pkg/database/migrations.go:75
1764845495230154 migrating to version 3... /build/pkg/database/migrations.go:82
1764845495230159 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764845495230168 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764845495230182 migrating to version 4... /build/pkg/database/migrations.go:89
1764845495230187 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764845495230198 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764845495230204 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764845495230219 migrating to version 5... /build/pkg/database/migrations.go:96
1764845495230224 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764845495230232 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764845495230237 no events need re-encoding /build/pkg/database/migrations.go:652
1764845495230250 migrating to version 6... /build/pkg/database/migrations.go:103
1764845495230255 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764845495230265 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764845495230269 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 10:51:35 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 3.737037757s
Events/sec: 13379.58
Avg latency: 1.744659ms
P90 latency: 2.47401ms
P95 latency: 2.895953ms
P99 latency: 4.909556ms
Bottom 10% Avg latency: 897.762µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 421.882059ms
Burst completed: 5000 events in 412.531799ms
Burst completed: 5000 events in 429.098267ms
Burst completed: 5000 events in 390.670143ms
Burst completed: 5000 events in 438.603112ms
Burst completed: 5000 events in 366.944086ms
Burst completed: 5000 events in 534.455064ms
Burst completed: 5000 events in 559.621403ms
Burst completed: 5000 events in 393.427363ms
Burst completed: 5000 events in 371.875354ms
Burst test completed: 50000 events in 9.324705477s, errors: 0
Events/sec: 5362.10
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.924958418s
Combined ops/sec: 2006.02
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 244167 queries in 1m0.008740456s
Queries/sec: 4068.86
Avg query latency: 4.157543ms
P95 query latency: 15.724716ms
P99 query latency: 24.284362ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 227664 operations (177664 queries, 50000 writes) in 1m0.005538199s
Operations/sec: 3794.05
Avg latency: 2.523997ms
Avg query latency: 2.668863ms
Avg write latency: 2.009247ms
P95 latency: 7.235855ms
P99 latency: 20.657306ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 3.737037757s
Total Events: 50000
Events/sec: 13379.58
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 217 MB
Avg Latency: 1.744659ms
P90 Latency: 2.47401ms
P95 Latency: 2.895953ms
P99 Latency: 4.909556ms
Bottom 10% Avg Latency: 897.762µs
----------------------------------------
Test: Burst Pattern
Duration: 9.324705477s
Total Events: 50000
Events/sec: 5362.10
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 304 MB
Avg Latency: 2.063122ms
P90 Latency: 3.130188ms
P95 Latency: 3.8975ms
P99 Latency: 6.378352ms
Bottom 10% Avg Latency: 954.959µs
----------------------------------------
Test: Mixed Read/Write
Duration: 24.924958418s
Total Events: 50000
Events/sec: 2006.02
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 272 MB
Avg Latency: 475.177µs
P90 Latency: 996.497µs
P95 Latency: 1.205595ms
P99 Latency: 1.873106ms
Bottom 10% Avg Latency: 1.414397ms
----------------------------------------
Test: Query Performance
Duration: 1m0.008740456s
Total Events: 244167
Events/sec: 4068.86
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 148 MB
Avg Latency: 4.157543ms
P90 Latency: 12.228439ms
P95 Latency: 15.724716ms
P99 Latency: 24.284362ms
Bottom 10% Avg Latency: 17.427943ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.005538199s
Total Events: 227664
Events/sec: 3794.05
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 211 MB
Avg Latency: 2.523997ms
P90 Latency: 5.269722ms
P95 Latency: 7.235855ms
P99 Latency: 20.657306ms
Bottom 10% Avg Latency: 10.288906ms
----------------------------------------
Report saved to: /tmp/benchmark_next-orly-neo4j_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_next-orly-neo4j_8/benchmark_report.adoc
RELAY_NAME: next-orly-neo4j
RELAY_URL: ws://next-orly-neo4j:8080
TEST_TIMESTAMP: 2025-12-04T10:54:54+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -0,0 +1,201 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_nostr-rs-relay_8
Events: 50000, Workers: 24, Duration: 1m0s
1764846517510492 migrating to version 1... /build/pkg/database/migrations.go:68
1764846517510692 migrating to version 2... /build/pkg/database/migrations.go:75
1764846517511210 migrating to version 3... /build/pkg/database/migrations.go:82
1764846517511251 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764846517511274 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764846517511304 migrating to version 4... /build/pkg/database/migrations.go:89
1764846517511317 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764846517511329 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764846517511340 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764846517511366 migrating to version 5... /build/pkg/database/migrations.go:96
1764846517511373 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764846517511388 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764846517511394 no events need re-encoding /build/pkg/database/migrations.go:652
1764846517511443 migrating to version 6... /build/pkg/database/migrations.go:103
1764846517511452 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764846517511466 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764846517511472 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 11:08:37 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 4.118969633s
Events/sec: 12138.96
Avg latency: 1.937994ms
P90 latency: 2.852802ms
P95 latency: 3.444328ms
P99 latency: 5.727836ms
Bottom 10% Avg latency: 946.456µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 403.020917ms
Burst completed: 5000 events in 372.371612ms
Burst completed: 5000 events in 424.238707ms
Burst completed: 5000 events in 385.317421ms
Burst completed: 5000 events in 516.841571ms
Burst completed: 5000 events in 591.703187ms
Burst completed: 5000 events in 445.314485ms
Burst completed: 5000 events in 374.011153ms
Burst completed: 5000 events in 398.6942ms
Burst completed: 5000 events in 365.965806ms
Burst test completed: 50000 events in 9.28457886s, errors: 0
Events/sec: 5385.27
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.684808581s
Combined ops/sec: 2025.54
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 251672 queries in 1m0.006178379s
Queries/sec: 4194.10
Avg query latency: 4.01666ms
P95 query latency: 15.051188ms
P99 query latency: 22.451758ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 219001 operations (169001 queries, 50000 writes) in 1m0.004144652s
Operations/sec: 3649.76
Avg latency: 2.620549ms
Avg query latency: 2.844617ms
Avg write latency: 1.863195ms
P95 latency: 7.482377ms
P99 latency: 20.396275ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 4.118969633s
Total Events: 50000
Events/sec: 12138.96
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 150 MB
Avg Latency: 1.937994ms
P90 Latency: 2.852802ms
P95 Latency: 3.444328ms
P99 Latency: 5.727836ms
Bottom 10% Avg Latency: 946.456µs
----------------------------------------
Test: Burst Pattern
Duration: 9.28457886s
Total Events: 50000
Events/sec: 5385.27
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 259 MB
Avg Latency: 2.040218ms
P90 Latency: 3.113648ms
P95 Latency: 3.901749ms
P99 Latency: 6.623842ms
Bottom 10% Avg Latency: 930.455µs
----------------------------------------
Test: Mixed Read/Write
Duration: 24.684808581s
Total Events: 50000
Events/sec: 2025.54
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 170 MB
Avg Latency: 435.806µs
P90 Latency: 909.692µs
P95 Latency: 1.063135ms
P99 Latency: 1.414473ms
Bottom 10% Avg Latency: 1.173081ms
----------------------------------------
Test: Query Performance
Duration: 1m0.006178379s
Total Events: 251672
Events/sec: 4194.10
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 159 MB
Avg Latency: 4.01666ms
P90 Latency: 11.874709ms
P95 Latency: 15.051188ms
P99 Latency: 22.451758ms
Bottom 10% Avg Latency: 16.47537ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.004144652s
Total Events: 219001
Events/sec: 3649.76
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 154 MB
Avg Latency: 2.620549ms
P90 Latency: 5.591506ms
P95 Latency: 7.482377ms
P99 Latency: 20.396275ms
Bottom 10% Avg Latency: 10.345145ms
----------------------------------------
Report saved to: /tmp/benchmark_nostr-rs-relay_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_nostr-rs-relay_8/benchmark_report.adoc
RELAY_NAME: nostr-rs-relay
RELAY_URL: ws://nostr-rs-relay:8080
TEST_TIMESTAMP: 2025-12-04T11:11:56+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -0,0 +1,201 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_relayer-basic_8
Events: 50000, Workers: 24, Duration: 1m0s
1764846109277147 migrating to version 1... /build/pkg/database/migrations.go:68
1764846109277265 migrating to version 2... /build/pkg/database/migrations.go:75
1764846109277319 migrating to version 3... /build/pkg/database/migrations.go:82
1764846109277325 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764846109277335 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764846109277350 migrating to version 4... /build/pkg/database/migrations.go:89
1764846109277355 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764846109277363 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764846109277369 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764846109277389 migrating to version 5... /build/pkg/database/migrations.go:96
1764846109277396 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764846109277405 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764846109277410 no events need re-encoding /build/pkg/database/migrations.go:652
1764846109277424 migrating to version 6... /build/pkg/database/migrations.go:103
1764846109277429 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764846109277439 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764846109277443 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 11:01:49 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 3.829064715s
Events/sec: 13058.02
Avg latency: 1.792879ms
P90 latency: 2.621872ms
P95 latency: 3.153103ms
P99 latency: 4.914106ms
Bottom 10% Avg latency: 919.64µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 406.089196ms
Burst completed: 5000 events in 571.162214ms
Burst completed: 5000 events in 417.21044ms
Burst completed: 5000 events in 388.695149ms
Burst completed: 5000 events in 448.68702ms
Burst completed: 5000 events in 349.680067ms
Burst completed: 5000 events in 352.379547ms
Burst completed: 5000 events in 348.007743ms
Burst completed: 5000 events in 396.819076ms
Burst completed: 5000 events in 388.190088ms
Burst test completed: 50000 events in 9.077665116s, errors: 0
Events/sec: 5508.02
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.750507885s
Combined ops/sec: 2020.16
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 272535 queries in 1m0.006407297s
Queries/sec: 4541.76
Avg query latency: 3.702484ms
P95 query latency: 14.064278ms
P99 query latency: 21.546984ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 236255 operations (186255 queries, 50000 writes) in 1m0.005350378s
Operations/sec: 3937.23
Avg latency: 2.284443ms
Avg query latency: 2.471631ms
Avg write latency: 1.58715ms
P95 latency: 6.469447ms
P99 latency: 17.551758ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 3.829064715s
Total Events: 50000
Events/sec: 13058.02
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 226 MB
Avg Latency: 1.792879ms
P90 Latency: 2.621872ms
P95 Latency: 3.153103ms
P99 Latency: 4.914106ms
Bottom 10% Avg Latency: 919.64µs
----------------------------------------
Test: Burst Pattern
Duration: 9.077665116s
Total Events: 50000
Events/sec: 5508.02
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 263 MB
Avg Latency: 1.938961ms
P90 Latency: 2.872088ms
P95 Latency: 3.585166ms
P99 Latency: 6.443979ms
Bottom 10% Avg Latency: 919.151µs
----------------------------------------
Test: Mixed Read/Write
Duration: 24.750507885s
Total Events: 50000
Events/sec: 2020.16
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 173 MB
Avg Latency: 448.262µs
P90 Latency: 942.865µs
P95 Latency: 1.09768ms
P99 Latency: 1.554199ms
Bottom 10% Avg Latency: 1.241163ms
----------------------------------------
Test: Query Performance
Duration: 1m0.006407297s
Total Events: 272535
Events/sec: 4541.76
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 134 MB
Avg Latency: 3.702484ms
P90 Latency: 10.940029ms
P95 Latency: 14.064278ms
P99 Latency: 21.546984ms
Bottom 10% Avg Latency: 15.564533ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.005350378s
Total Events: 236255
Events/sec: 3937.23
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 200 MB
Avg Latency: 2.284443ms
P90 Latency: 4.876796ms
P95 Latency: 6.469447ms
P99 Latency: 17.551758ms
Bottom 10% Avg Latency: 8.957464ms
----------------------------------------
Report saved to: /tmp/benchmark_relayer-basic_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_relayer-basic_8/benchmark_report.adoc
RELAY_NAME: relayer-basic
RELAY_URL: ws://relayer-basic:7447
TEST_TIMESTAMP: 2025-12-04T11:05:08+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -0,0 +1,63 @@
Starting Network Graph Traversal Benchmark
Relay URL: ws://rely-sqlite:3334
Workers: 24
Pubkeys: 100000, Follows per pubkey: 1-1000
╔════════════════════════════════════════════════════════╗
║ NETWORK GRAPH TRAVERSAL BENCHMARK (100k Pubkeys) ║
║ Relay: ws://rely-sqlite:3334 ║
╚════════════════════════════════════════════════════════╝
Generating 100000 deterministic pubkeys...
2025/12/04 11:12:01 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Generated 10000/100000 pubkeys...
Generated 20000/100000 pubkeys...
Generated 30000/100000 pubkeys...
Generated 40000/100000 pubkeys...
Generated 50000/100000 pubkeys...
Generated 60000/100000 pubkeys...
Generated 70000/100000 pubkeys...
Generated 80000/100000 pubkeys...
Generated 90000/100000 pubkeys...
Generated 100000/100000 pubkeys...
Generated 100000 pubkeys in 2.699112464s
Generating follow graph (1-1000 follows per pubkey)...
Generated follow lists for 10000/100000 pubkeys...
Generated follow lists for 20000/100000 pubkeys...
Generated follow lists for 30000/100000 pubkeys...
Generated follow lists for 40000/100000 pubkeys...
Generated follow lists for 50000/100000 pubkeys...
Generated follow lists for 60000/100000 pubkeys...
Generated follow lists for 70000/100000 pubkeys...
Generated follow lists for 80000/100000 pubkeys...
Generated follow lists for 90000/100000 pubkeys...
Generated follow lists for 100000/100000 pubkeys...
Generated follow graph in 5.172393834s (avg 500.5 follows/pubkey, total 50048088 follows)
Connecting to relay: ws://rely-sqlite:3334
Connected successfully!
Creating follow list events via WebSocket...
Queued 10000/100000 follow list events...
Queued 20000/100000 follow list events...
Queued 30000/100000 follow list events...
Queued 40000/100000 follow list events...
Queued 50000/100000 follow list events...
Queued 60000/100000 follow list events...
Queued 70000/100000 follow list events...
Queued 80000/100000 follow list events...
Queued 90000/100000 follow list events...
Queued 100000/100000 follow list events...
Created 100000 follow list events in 1m47.750797847s (928.07 events/sec, errors: 0)
Avg latency: 5.218453ms, P95: 30.619168ms, P99: 66.455368ms
Waiting for events to be processed...
=== Third-Degree Graph Traversal Benchmark (Network) ===
Traversing 3 degrees of follows via WebSocket...
Sampling 1000 pubkeys for traversal...
Killed
RELAY_NAME: rely-sqlite
RELAY_URL: ws://rely-sqlite:3334
TEST_TYPE: Graph Traversal
STATUS: FAILED
TEST_TIMESTAMP: 2025-12-04T13:18:55+00:00

View File

@@ -0,0 +1,202 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_rely-sqlite_8
Events: 50000, Workers: 24, Duration: 1m0s
1764845084601162 migrating to version 1... /build/pkg/database/migrations.go:68
1764845084601278 migrating to version 2... /build/pkg/database/migrations.go:75
1764845084601338 migrating to version 3... /build/pkg/database/migrations.go:82
1764845084601353 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764845084601368 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764845084601398 migrating to version 4... /build/pkg/database/migrations.go:89
1764845084601404 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764845084601425 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764845084601432 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764845084601453 migrating to version 5... /build/pkg/database/migrations.go:96
1764845084601459 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764845084601470 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764845084601476 no events need re-encoding /build/pkg/database/migrations.go:652
1764845084601492 migrating to version 6... /build/pkg/database/migrations.go:103
1764845084601498 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764845084601512 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764845084601518 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 10:44:44 INFO: Extracted embedded libsecp256k1 to /tmp/orly-libsecp256k1/libsecp256k1.so
2025/12/04 10:44:44 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 4.863868097s
Events/sec: 10279.88
Avg latency: 2.303586ms
P90 latency: 3.506294ms
P95 latency: 4.26606ms
P99 latency: 6.589692ms
Bottom 10% Avg latency: 1.039748ms
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 490.290781ms
Burst completed: 5000 events in 660.13017ms
Burst completed: 5000 events in 395.417016ms
Burst completed: 5000 events in 386.572933ms
Burst completed: 5000 events in 453.417446ms
Burst completed: 5000 events in 431.074552ms
Burst completed: 5000 events in 425.56635ms
Burst completed: 5000 events in 480.609672ms
Burst completed: 5000 events in 491.483839ms
Burst completed: 5000 events in 855.851556ms
Burst test completed: 50000 events in 10.076554319s, errors: 0
Events/sec: 4962.01
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.99725206s
Combined ops/sec: 2000.22
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 248134 queries in 1m0.010897965s
Queries/sec: 4134.82
Avg query latency: 4.008215ms
P95 query latency: 15.241611ms
P99 query latency: 23.364071ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 223423 operations (173423 queries, 50000 writes) in 1m0.003723611s
Operations/sec: 3723.49
Avg latency: 2.490436ms
Avg query latency: 2.752076ms
Avg write latency: 1.582945ms
P95 latency: 7.431916ms
P99 latency: 18.31948ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 4.863868097s
Total Events: 50000
Events/sec: 10279.88
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 210 MB
Avg Latency: 2.303586ms
P90 Latency: 3.506294ms
P95 Latency: 4.26606ms
P99 Latency: 6.589692ms
Bottom 10% Avg Latency: 1.039748ms
----------------------------------------
Test: Burst Pattern
Duration: 10.076554319s
Total Events: 50000
Events/sec: 4962.01
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 306 MB
Avg Latency: 2.440058ms
P90 Latency: 3.974234ms
P95 Latency: 5.200288ms
P99 Latency: 9.335708ms
Bottom 10% Avg Latency: 1.00845ms
----------------------------------------
Test: Mixed Read/Write
Duration: 24.99725206s
Total Events: 50000
Events/sec: 2000.22
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 270 MB
Avg Latency: 457.992µs
P90 Latency: 957.983µs
P95 Latency: 1.136012ms
P99 Latency: 1.617368ms
Bottom 10% Avg Latency: 1.292479ms
----------------------------------------
Test: Query Performance
Duration: 1m0.010897965s
Total Events: 248134
Events/sec: 4134.82
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 138 MB
Avg Latency: 4.008215ms
P90 Latency: 11.8477ms
P95 Latency: 15.241611ms
P99 Latency: 23.364071ms
Bottom 10% Avg Latency: 16.87008ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.003723611s
Total Events: 223423
Events/sec: 3723.49
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 195 MB
Avg Latency: 2.490436ms
P90 Latency: 5.497334ms
P95 Latency: 7.431916ms
P99 Latency: 18.31948ms
Bottom 10% Avg Latency: 9.827857ms
----------------------------------------
Report saved to: /tmp/benchmark_rely-sqlite_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_rely-sqlite_8/benchmark_report.adoc
RELAY_NAME: rely-sqlite
RELAY_URL: ws://rely-sqlite:3334
TEST_TIMESTAMP: 2025-12-04T10:48:05+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

View File

@@ -0,0 +1,201 @@
Starting Nostr Relay Benchmark (Badger Backend)
Data Directory: /tmp/benchmark_strfry_8
Events: 50000, Workers: 24, Duration: 1m0s
1764846313173994 migrating to version 1... /build/pkg/database/migrations.go:68
1764846313174100 migrating to version 2... /build/pkg/database/migrations.go:75
1764846313174135 migrating to version 3... /build/pkg/database/migrations.go:82
1764846313174143 cleaning up ephemeral events (kinds 20000-29999)... /build/pkg/database/migrations.go:304
1764846313174154 cleaned up 0 ephemeral events from database /build/pkg/database/migrations.go:349
1764846313174172 migrating to version 4... /build/pkg/database/migrations.go:89
1764846313174177 converting events to optimized inline storage (Reiser4 optimization)... /build/pkg/database/migrations.go:357
1764846313174193 found 0 events to convert (0 regular, 0 replaceable, 0 addressable) /build/pkg/database/migrations.go:446
1764846313174199 migration complete: converted 0 events to optimized inline storage, deleted 0 old keys /build/pkg/database/migrations.go:555
1764846313174215 migrating to version 5... /build/pkg/database/migrations.go:96
1764846313174222 re-encoding events with optimized tag binary format... /build/pkg/database/migrations.go:562
1764846313174232 found 0 events with e/p tags to re-encode /build/pkg/database/migrations.go:649
1764846313174238 no events need re-encoding /build/pkg/database/migrations.go:652
1764846313174259 migrating to version 6... /build/pkg/database/migrations.go:103
1764846313174264 converting events to compact serial-reference format... /build/pkg/database/migrations.go:706
1764846313174274 found 0 events to convert to compact format /build/pkg/database/migrations.go:846
1764846313174282 no events need conversion /build/pkg/database/migrations.go:849
╔════════════════════════════════════════════════════════╗
║ BADGER BACKEND BENCHMARK SUITE ║
╚════════════════════════════════════════════════════════╝
=== Starting Badger benchmark ===
RunPeakThroughputTest (Badger)..
=== Peak Throughput Test ===
2025/12/04 11:05:13 INFO: Successfully loaded embedded libsecp256k1 v5.0.0 from /tmp/orly-libsecp256k1/libsecp256k1.so
Events saved: 50000/50000 (100.0%), errors: 0
Duration: 3.876849434s
Events/sec: 12897.07
Avg latency: 1.815658ms
P90 latency: 2.61564ms
P95 latency: 3.107597ms
P99 latency: 5.258081ms
Bottom 10% Avg latency: 919.54µs
Wiping database between tests...
RunBurstPatternTest (Badger)..
=== Burst Pattern Test ===
Burst completed: 5000 events in 399.187129ms
Burst completed: 5000 events in 388.99822ms
Burst completed: 5000 events in 402.825697ms
Burst completed: 5000 events in 402.426226ms
Burst completed: 5000 events in 509.746009ms
Burst completed: 5000 events in 360.327121ms
Burst completed: 5000 events in 354.620576ms
Burst completed: 5000 events in 340.233233ms
Burst completed: 5000 events in 484.991889ms
Burst completed: 5000 events in 450.540384ms
Burst test completed: 50000 events in 9.101582141s, errors: 0
Events/sec: 5493.55
Wiping database between tests...
RunMixedReadWriteTest (Badger)..
=== Mixed Read/Write Test ===
Generating 1000 unique synthetic events (minimum 300 bytes each)...
Generated 1000 events:
Average content size: 312 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database for read tests...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Mixed test completed: 25000 writes, 25000 reads in 24.968859674s
Combined ops/sec: 2002.49
Wiping database between tests...
RunQueryTest (Badger)..
=== Query Test ===
Generating 10000 unique synthetic events (minimum 300 bytes each)...
Generated 10000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 10000 events for query tests...
Query test completed: 261904 queries in 1m0.006069229s
Queries/sec: 4364.63
Avg query latency: 3.860709ms
P95 query latency: 14.612102ms
P99 query latency: 22.708667ms
Wiping database between tests...
RunConcurrentQueryStoreTest (Badger)..
=== Concurrent Query/Store Test ===
Generating 5000 unique synthetic events (minimum 300 bytes each)...
Generated 5000 events:
Average content size: 313 bytes
All events are unique (incremental timestamps)
All events are properly signed
Pre-populating database with 5000 events for concurrent query/store test...
Generating 50000 unique synthetic events (minimum 300 bytes each)...
Generated 50000 events:
Average content size: 314 bytes
All events are unique (incremental timestamps)
All events are properly signed
Concurrent test completed: 230898 operations (180898 queries, 50000 writes) in 1m0.007085265s
Operations/sec: 3847.85
Avg latency: 2.400221ms
Avg query latency: 2.609803ms
Avg write latency: 1.641962ms
P95 latency: 6.834352ms
P99 latency: 18.125521ms
=== Badger benchmark completed ===
================================================================================
BENCHMARK REPORT
================================================================================
Test: Peak Throughput
Duration: 3.876849434s
Total Events: 50000
Events/sec: 12897.07
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 194 MB
Avg Latency: 1.815658ms
P90 Latency: 2.61564ms
P95 Latency: 3.107597ms
P99 Latency: 5.258081ms
Bottom 10% Avg Latency: 919.54µs
----------------------------------------
Test: Burst Pattern
Duration: 9.101582141s
Total Events: 50000
Events/sec: 5493.55
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 189 MB
Avg Latency: 1.954573ms
P90 Latency: 2.922786ms
P95 Latency: 3.66591ms
P99 Latency: 6.353176ms
Bottom 10% Avg Latency: 904.101µs
----------------------------------------
Test: Mixed Read/Write
Duration: 24.968859674s
Total Events: 50000
Events/sec: 2002.49
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 188 MB
Avg Latency: 443.895µs
P90 Latency: 930.312µs
P95 Latency: 1.08191ms
P99 Latency: 1.476191ms
Bottom 10% Avg Latency: 1.222569ms
----------------------------------------
Test: Query Performance
Duration: 1m0.006069229s
Total Events: 261904
Events/sec: 4364.63
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 198 MB
Avg Latency: 3.860709ms
P90 Latency: 11.381821ms
P95 Latency: 14.612102ms
P99 Latency: 22.708667ms
Bottom 10% Avg Latency: 16.28305ms
----------------------------------------
Test: Concurrent Query/Store
Duration: 1m0.007085265s
Total Events: 230898
Events/sec: 3847.85
Success Rate: 100.0%
Concurrent Workers: 24
Memory Used: 178 MB
Avg Latency: 2.400221ms
P90 Latency: 5.16819ms
P95 Latency: 6.834352ms
P99 Latency: 18.125521ms
Bottom 10% Avg Latency: 9.340478ms
----------------------------------------
Report saved to: /tmp/benchmark_strfry_8/benchmark_report.txt
AsciiDoc report saved to: /tmp/benchmark_strfry_8/benchmark_report.adoc
RELAY_NAME: strfry
RELAY_URL: ws://strfry:8080
TEST_TIMESTAMP: 2025-12-04T11:08:32+00:00
BENCHMARK_CONFIG:
Events: 50000
Workers: 24
Duration: 60s

4
go.mod
View File

@@ -3,7 +3,7 @@ module next.orly.dev
go 1.25.3
require (
git.mleku.dev/mleku/nostr v1.0.7
git.mleku.dev/mleku/nostr v1.0.8
github.com/adrg/xdg v0.5.3
github.com/aperturerobotics/go-indexeddb v0.2.3
github.com/dgraph-io/badger/v4 v4.8.0
@@ -82,5 +82,3 @@ require (
)
retract v1.0.3
replace git.mleku.dev/mleku/nostr => /home/mleku/src/git.mleku.dev/mleku/nostr

2
go.sum
View File

@@ -1,3 +1,5 @@
git.mleku.dev/mleku/nostr v1.0.8 h1:YYREdIxobEqYkzxQ7/5ALACPzLkiHW+CTira+VvSQZk=
git.mleku.dev/mleku/nostr v1.0.8/go.mod h1:iYTlg2WKJXJ0kcsM6QBGOJ0UDiJidMgL/i64cHyPjZc=
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 h1:ClzzXMDDuUbWfNNZqGeYq4PnYOlwlOVIvSyNaIy0ykg=

View File

@@ -0,0 +1,24 @@
# Docker Compose file for Neo4j test database
# Usage: docker compose up -d && go test ./pkg/neo4j/... && docker compose down
services:
neo4j-test:
image: neo4j:5.15.0-community
container_name: neo4j-test
ports:
- "7687:7687" # Bolt protocol
- "7474:7474" # HTTP (browser interface)
environment:
- NEO4J_AUTH=neo4j/testpassword
- NEO4J_PLUGINS=["apoc"]
- NEO4J_dbms_security_procedures_unrestricted=apoc.*
- NEO4J_dbms_memory_heap_initial__size=512m
- NEO4J_dbms_memory_heap_max__size=1g
- NEO4J_dbms_memory_pagecache_size=512m
healthcheck:
test: ["CMD", "cypher-shell", "-u", "neo4j", "-p", "testpassword", "RETURN 1"]
interval: 5s
timeout: 10s
retries: 10
start_period: 30s
tmpfs:
- /data # Use tmpfs for faster tests

277
pkg/neo4j/hex_utils_test.go Normal file
View File

@@ -0,0 +1,277 @@
package neo4j
import (
"testing"
"git.mleku.dev/mleku/nostr/encoders/tag"
)
// TestIsBinaryEncoded tests the IsBinaryEncoded function
func TestIsBinaryEncoded(t *testing.T) {
tests := []struct {
name string
input []byte
expected bool
}{
{
name: "Valid binary encoded (33 bytes with null terminator)",
input: append(make([]byte, 32), 0),
expected: true,
},
{
name: "Invalid - 32 bytes without terminator",
input: make([]byte, 32),
expected: false,
},
{
name: "Invalid - 33 bytes without null terminator",
input: append(make([]byte, 32), 1),
expected: false,
},
{
name: "Invalid - 64 bytes (hex string)",
input: []byte("0000000000000000000000000000000000000000000000000000000000000001"),
expected: false,
},
{
name: "Invalid - empty",
input: []byte{},
expected: false,
},
{
name: "Invalid - too short",
input: []byte{0, 1, 2, 3},
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := IsBinaryEncoded(tt.input)
if result != tt.expected {
t.Errorf("IsBinaryEncoded(%v) = %v, want %v", tt.input, result, tt.expected)
}
})
}
}
// TestNormalizePubkeyHex tests the NormalizePubkeyHex function
func TestNormalizePubkeyHex(t *testing.T) {
// Create a 32-byte test value
testBytes := make([]byte, 32)
testBytes[31] = 0x01 // Set last byte to 1
// Create binary-encoded version (33 bytes with null terminator)
binaryEncoded := append(testBytes, 0)
tests := []struct {
name string
input []byte
expected string
}{
{
name: "Binary encoded to hex",
input: binaryEncoded,
expected: "0000000000000000000000000000000000000000000000000000000000000001",
},
{
name: "Lowercase hex passthrough",
input: []byte("0000000000000000000000000000000000000000000000000000000000000001"),
expected: "0000000000000000000000000000000000000000000000000000000000000001",
},
{
name: "Uppercase hex to lowercase",
input: []byte("ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789"),
expected: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
},
{
name: "Mixed case hex to lowercase",
input: []byte("AbCdEf0123456789AbCdEf0123456789AbCdEf0123456789AbCdEf0123456789"),
expected: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
},
{
name: "Prefix hex (shorter than 64)",
input: []byte("ABCD"),
expected: "abcd",
},
{
name: "Empty input",
input: []byte{},
expected: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := NormalizePubkeyHex(tt.input)
if result != tt.expected {
t.Errorf("NormalizePubkeyHex(%v) = %q, want %q", tt.input, result, tt.expected)
}
})
}
}
// TestExtractPTagValue tests the ExtractPTagValue function
func TestExtractPTagValue(t *testing.T) {
// Create a valid pubkey hex string
validHex := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
tests := []struct {
name string
tag *tag.T
expected string
}{
{
name: "Nil tag",
tag: nil,
expected: "",
},
{
name: "Empty tag",
tag: &tag.T{T: [][]byte{}},
expected: "",
},
{
name: "Tag with only key",
tag: &tag.T{T: [][]byte{[]byte("p")}},
expected: "",
},
{
name: "Valid p-tag with hex value",
tag: &tag.T{T: [][]byte{
[]byte("p"),
[]byte(validHex),
}},
expected: validHex,
},
{
name: "P-tag with uppercase hex",
tag: &tag.T{T: [][]byte{
[]byte("p"),
[]byte("ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789"),
}},
expected: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := ExtractPTagValue(tt.tag)
if result != tt.expected {
t.Errorf("ExtractPTagValue() = %q, want %q", result, tt.expected)
}
})
}
}
// TestExtractETagValue tests the ExtractETagValue function
func TestExtractETagValue(t *testing.T) {
// Create a valid event ID hex string
validHex := "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"
tests := []struct {
name string
tag *tag.T
expected string
}{
{
name: "Nil tag",
tag: nil,
expected: "",
},
{
name: "Empty tag",
tag: &tag.T{T: [][]byte{}},
expected: "",
},
{
name: "Tag with only key",
tag: &tag.T{T: [][]byte{[]byte("e")}},
expected: "",
},
{
name: "Valid e-tag with hex value",
tag: &tag.T{T: [][]byte{
[]byte("e"),
[]byte(validHex),
}},
expected: validHex,
},
{
name: "E-tag with uppercase hex",
tag: &tag.T{T: [][]byte{
[]byte("e"),
[]byte("1234567890ABCDEF1234567890ABCDEF1234567890ABCDEF1234567890ABCDEF"),
}},
expected: "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := ExtractETagValue(tt.tag)
if result != tt.expected {
t.Errorf("ExtractETagValue() = %q, want %q", result, tt.expected)
}
})
}
}
// TestIsValidHexPubkey tests the IsValidHexPubkey function
func TestIsValidHexPubkey(t *testing.T) {
tests := []struct {
name string
input string
expected bool
}{
{
name: "Valid lowercase hex",
input: "0000000000000000000000000000000000000000000000000000000000000001",
expected: true,
},
{
name: "Valid uppercase hex",
input: "ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789",
expected: true,
},
{
name: "Valid mixed case hex",
input: "AbCdEf0123456789AbCdEf0123456789AbCdEf0123456789AbCdEf0123456789",
expected: true,
},
{
name: "Too short",
input: "0000000000000000000000000000000000000000000000000000000000000",
expected: false,
},
{
name: "Too long",
input: "00000000000000000000000000000000000000000000000000000000000000001",
expected: false,
},
{
name: "Contains non-hex character",
input: "000000000000000000000000000000000000000000000000000000000000000g",
expected: false,
},
{
name: "Empty string",
input: "",
expected: false,
},
{
name: "Contains space",
input: "0000000000000000000000000000000000000000000000000000000000000 01",
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := IsValidHexPubkey(tt.input)
if result != tt.expected {
t.Errorf("IsValidHexPubkey(%q) = %v, want %v", tt.input, result, tt.expected)
}
})
}
}

View File

@@ -20,6 +20,11 @@ var migrations = []Migration{
Description: "Merge Author nodes into NostrUser nodes",
Migrate: migrateAuthorToNostrUser,
},
{
Version: "v2",
Description: "Clean up binary-encoded pubkeys and event IDs to lowercase hex",
Migrate: migrateBinaryToHex,
},
}
// RunMigrations executes all pending migrations
@@ -195,3 +200,146 @@ func migrateAuthorToNostrUser(ctx context.Context, n *N) error {
n.Logger.Infof("completed Author to NostrUser migration")
return nil
}
// migrateBinaryToHex cleans up any binary-encoded pubkeys and event IDs
// The nostr library stores e/p tag values in binary format (33 bytes with null terminator),
// but Neo4j should store them as lowercase hex strings for consistent querying.
// This migration:
// 1. Finds NostrUser nodes with invalid (non-hex) pubkeys and deletes them
// 2. Finds Event nodes with invalid pubkeys/IDs and deletes them
// 3. Finds Tag nodes (type 'e' or 'p') with invalid values and deletes them
// 4. Cleans up MENTIONS relationships pointing to invalid NostrUser nodes
func migrateBinaryToHex(ctx context.Context, n *N) error {
// Step 1: Count problematic nodes before cleanup
n.Logger.Infof("scanning for binary-encoded values in Neo4j...")
// Check for NostrUser nodes with invalid pubkeys (not 64 char hex)
// A valid hex pubkey is exactly 64 lowercase hex characters
countInvalidUsersCypher := `
MATCH (u:NostrUser)
WHERE size(u.pubkey) <> 64
OR NOT u.pubkey =~ '^[0-9a-f]{64}$'
RETURN count(u) AS count
`
result, err := n.ExecuteRead(ctx, countInvalidUsersCypher, nil)
if err != nil {
return fmt.Errorf("failed to count invalid NostrUser nodes: %w", err)
}
var invalidUserCount int64
if result.Next(ctx) {
if count, ok := result.Record().Values[0].(int64); ok {
invalidUserCount = count
}
}
n.Logger.Infof("found %d NostrUser nodes with invalid pubkeys", invalidUserCount)
// Check for Event nodes with invalid pubkeys or IDs
countInvalidEventsCypher := `
MATCH (e:Event)
WHERE (size(e.pubkey) <> 64 OR NOT e.pubkey =~ '^[0-9a-f]{64}$')
OR (size(e.id) <> 64 OR NOT e.id =~ '^[0-9a-f]{64}$')
RETURN count(e) AS count
`
result, err = n.ExecuteRead(ctx, countInvalidEventsCypher, nil)
if err != nil {
return fmt.Errorf("failed to count invalid Event nodes: %w", err)
}
var invalidEventCount int64
if result.Next(ctx) {
if count, ok := result.Record().Values[0].(int64); ok {
invalidEventCount = count
}
}
n.Logger.Infof("found %d Event nodes with invalid pubkeys or IDs", invalidEventCount)
// Check for Tag nodes (e/p type) with invalid values
countInvalidTagsCypher := `
MATCH (t:Tag)
WHERE t.type IN ['e', 'p']
AND (size(t.value) <> 64 OR NOT t.value =~ '^[0-9a-f]{64}$')
RETURN count(t) AS count
`
result, err = n.ExecuteRead(ctx, countInvalidTagsCypher, nil)
if err != nil {
return fmt.Errorf("failed to count invalid Tag nodes: %w", err)
}
var invalidTagCount int64
if result.Next(ctx) {
if count, ok := result.Record().Values[0].(int64); ok {
invalidTagCount = count
}
}
n.Logger.Infof("found %d Tag nodes (e/p type) with invalid values", invalidTagCount)
// If nothing to clean up, we're done
if invalidUserCount == 0 && invalidEventCount == 0 && invalidTagCount == 0 {
n.Logger.Infof("no binary-encoded values found, migration complete")
return nil
}
// Step 2: Delete invalid NostrUser nodes and their relationships
if invalidUserCount > 0 {
n.Logger.Infof("deleting %d invalid NostrUser nodes...", invalidUserCount)
deleteInvalidUsersCypher := `
MATCH (u:NostrUser)
WHERE size(u.pubkey) <> 64
OR NOT u.pubkey =~ '^[0-9a-f]{64}$'
DETACH DELETE u
`
_, err = n.ExecuteWrite(ctx, deleteInvalidUsersCypher, nil)
if err != nil {
return fmt.Errorf("failed to delete invalid NostrUser nodes: %w", err)
}
n.Logger.Infof("deleted %d invalid NostrUser nodes", invalidUserCount)
}
// Step 3: Delete invalid Event nodes and their relationships
if invalidEventCount > 0 {
n.Logger.Infof("deleting %d invalid Event nodes...", invalidEventCount)
deleteInvalidEventsCypher := `
MATCH (e:Event)
WHERE (size(e.pubkey) <> 64 OR NOT e.pubkey =~ '^[0-9a-f]{64}$')
OR (size(e.id) <> 64 OR NOT e.id =~ '^[0-9a-f]{64}$')
DETACH DELETE e
`
_, err = n.ExecuteWrite(ctx, deleteInvalidEventsCypher, nil)
if err != nil {
return fmt.Errorf("failed to delete invalid Event nodes: %w", err)
}
n.Logger.Infof("deleted %d invalid Event nodes", invalidEventCount)
}
// Step 4: Delete invalid Tag nodes (e/p type) and their relationships
if invalidTagCount > 0 {
n.Logger.Infof("deleting %d invalid Tag nodes...", invalidTagCount)
deleteInvalidTagsCypher := `
MATCH (t:Tag)
WHERE t.type IN ['e', 'p']
AND (size(t.value) <> 64 OR NOT t.value =~ '^[0-9a-f]{64}$')
DETACH DELETE t
`
_, err = n.ExecuteWrite(ctx, deleteInvalidTagsCypher, nil)
if err != nil {
return fmt.Errorf("failed to delete invalid Tag nodes: %w", err)
}
n.Logger.Infof("deleted %d invalid Tag nodes", invalidTagCount)
}
// Step 5: Clean up any orphaned MENTIONS/REFERENCES relationships
// These would be relationships pointing to nodes we just deleted
cleanupOrphanedCypher := `
// Clean up any ProcessedSocialEvent nodes with invalid pubkeys
MATCH (p:ProcessedSocialEvent)
WHERE size(p.pubkey) <> 64
OR NOT p.pubkey =~ '^[0-9a-f]{64}$'
DETACH DELETE p
`
_, _ = n.ExecuteWrite(ctx, cleanupOrphanedCypher, nil)
// Ignore errors - best effort cleanup
n.Logger.Infof("binary-to-hex migration completed successfully")
return nil
}

View File

@@ -0,0 +1,302 @@
package neo4j
import (
"context"
"testing"
)
// TestMigrationV2_CleanupBinaryEncodedValues tests that migration v2 properly
// cleans up binary-encoded pubkeys and event IDs
func TestMigrationV2_CleanupBinaryEncodedValues(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
ctx := context.Background()
// Create some valid NostrUser nodes (should NOT be deleted)
validPubkeys := []string{
"0000000000000000000000000000000000000000000000000000000000000001",
"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
}
for _, pk := range validPubkeys {
setupInvalidNostrUser(t, pk) // Using setupInvalidNostrUser to create directly
}
// Create some invalid NostrUser nodes (should be deleted)
invalidPubkeys := []string{
"binary\x00garbage\x00data", // Binary garbage
"ABCDEF", // Too short
"GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG", // Non-hex chars
string(append(make([]byte, 32), 0)), // 33-byte binary format
}
for _, pk := range invalidPubkeys {
setupInvalidNostrUser(t, pk)
}
// Verify invalid nodes exist before migration
invalidCountBefore := countInvalidNostrUsers(t)
if invalidCountBefore != 4 {
t.Errorf("Expected 4 invalid NostrUsers before migration, got %d", invalidCountBefore)
}
totalBefore := countNodes(t, "NostrUser")
if totalBefore != 6 {
t.Errorf("Expected 6 total NostrUsers before migration, got %d", totalBefore)
}
// Run the migration
err := migrateBinaryToHex(ctx, testDB)
if err != nil {
t.Fatalf("Migration failed: %v", err)
}
// Verify invalid nodes were deleted
invalidCountAfter := countInvalidNostrUsers(t)
if invalidCountAfter != 0 {
t.Errorf("Expected 0 invalid NostrUsers after migration, got %d", invalidCountAfter)
}
// Verify valid nodes were NOT deleted
totalAfter := countNodes(t, "NostrUser")
if totalAfter != 2 {
t.Errorf("Expected 2 valid NostrUsers after migration, got %d", totalAfter)
}
}
// TestMigrationV2_CleanupInvalidEvents tests that migration v2 properly
// cleans up Event nodes with invalid pubkeys or IDs
func TestMigrationV2_CleanupInvalidEvents(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
ctx := context.Background()
// Create valid events
validEventID := "1111111111111111111111111111111111111111111111111111111111111111"
validPubkey := "0000000000000000000000000000000000000000000000000000000000000001"
setupTestEvent(t, validEventID, validPubkey, 1, "[]")
// Create invalid events directly
setupInvalidEvent(t, "invalid_id", validPubkey) // Invalid ID
setupInvalidEvent(t, validEventID+"2", "invalid_pubkey") // Invalid pubkey (different ID to avoid duplicate)
setupInvalidEvent(t, "TOOSHORT", "binary\x00garbage") // Both invalid
// Count events before migration
eventsBefore := countNodes(t, "Event")
if eventsBefore != 4 {
t.Errorf("Expected 4 Events before migration, got %d", eventsBefore)
}
// Run the migration
err := migrateBinaryToHex(ctx, testDB)
if err != nil {
t.Fatalf("Migration failed: %v", err)
}
// Verify only valid event remains
eventsAfter := countNodes(t, "Event")
if eventsAfter != 1 {
t.Errorf("Expected 1 valid Event after migration, got %d", eventsAfter)
}
}
// TestMigrationV2_CleanupInvalidTags tests that migration v2 properly
// cleans up Tag nodes (e/p type) with invalid values
func TestMigrationV2_CleanupInvalidTags(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
ctx := context.Background()
// Create valid tags
validHex := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
setupInvalidTag(t, "e", validHex) // Valid e-tag
setupInvalidTag(t, "p", validHex) // Valid p-tag
setupInvalidTag(t, "t", "topic") // Non e/p tag (should not be affected)
// Create invalid e/p tags
setupInvalidTag(t, "e", "binary\x00garbage") // Invalid e-tag
setupInvalidTag(t, "p", "TOOSHORT") // Invalid p-tag (too short)
setupInvalidTag(t, "e", string(append(make([]byte, 32), 0))) // Binary encoded
// Count tags before migration
tagsBefore := countNodes(t, "Tag")
if tagsBefore != 6 {
t.Errorf("Expected 6 Tags before migration, got %d", tagsBefore)
}
invalidBefore := countInvalidTags(t)
if invalidBefore != 3 {
t.Errorf("Expected 3 invalid e/p Tags before migration, got %d", invalidBefore)
}
// Run the migration
err := migrateBinaryToHex(ctx, testDB)
if err != nil {
t.Fatalf("Migration failed: %v", err)
}
// Verify invalid tags were deleted
invalidAfter := countInvalidTags(t)
if invalidAfter != 0 {
t.Errorf("Expected 0 invalid e/p Tags after migration, got %d", invalidAfter)
}
// Verify valid tags remain (2 e/p valid + 1 t-tag)
tagsAfter := countNodes(t, "Tag")
if tagsAfter != 3 {
t.Errorf("Expected 3 Tags after migration, got %d", tagsAfter)
}
}
// TestMigrationV2_Idempotent tests that migration v2 can be run multiple times safely
func TestMigrationV2_Idempotent(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
ctx := context.Background()
// Create only valid data
validPubkey := "0000000000000000000000000000000000000000000000000000000000000001"
validEventID := "1111111111111111111111111111111111111111111111111111111111111111"
setupTestEvent(t, validEventID, validPubkey, 1, "[]")
countBefore := countNodes(t, "Event")
// Run migration first time
err := migrateBinaryToHex(ctx, testDB)
if err != nil {
t.Fatalf("First migration run failed: %v", err)
}
countAfterFirst := countNodes(t, "Event")
if countAfterFirst != countBefore {
t.Errorf("First migration changed valid event count: before=%d, after=%d", countBefore, countAfterFirst)
}
// Run migration second time
err = migrateBinaryToHex(ctx, testDB)
if err != nil {
t.Fatalf("Second migration run failed: %v", err)
}
countAfterSecond := countNodes(t, "Event")
if countAfterSecond != countBefore {
t.Errorf("Second migration changed valid event count: before=%d, after=%d", countBefore, countAfterSecond)
}
}
// TestMigrationV2_NoDataDoesNotFail tests that migration v2 succeeds with empty database
func TestMigrationV2_NoDataDoesNotFail(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up completely
cleanTestDatabase()
ctx := context.Background()
// Run migration on empty database - should not fail
err := migrateBinaryToHex(ctx, testDB)
if err != nil {
t.Fatalf("Migration on empty database failed: %v", err)
}
}
// TestMigrationMarking tests that migrations are properly tracked
func TestMigrationMarking(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
ctx := context.Background()
// Verify migration v2 has not been applied
if testDB.migrationApplied(ctx, "v2") {
t.Error("Migration v2 should not be applied before test")
}
// Mark migration as complete
err := testDB.markMigrationComplete(ctx, "v2", "Test migration")
if err != nil {
t.Fatalf("Failed to mark migration complete: %v", err)
}
// Verify migration is now marked as applied
if !testDB.migrationApplied(ctx, "v2") {
t.Error("Migration v2 should be applied after marking")
}
// Clean up
cleanTestDatabase()
}
// TestMigrationV1_AuthorToNostrUserMerge tests the author migration
func TestMigrationV1_AuthorToNostrUserMerge(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
ctx := context.Background()
// Create some Author nodes (legacy format)
authorPubkeys := []string{
"0000000000000000000000000000000000000000000000000000000000000001",
"0000000000000000000000000000000000000000000000000000000000000002",
}
for _, pk := range authorPubkeys {
cypher := `CREATE (a:Author {pubkey: $pubkey})`
_, err := testDB.ExecuteWrite(ctx, cypher, map[string]any{"pubkey": pk})
if err != nil {
t.Fatalf("Failed to create Author node: %v", err)
}
}
// Verify Author nodes exist
authorCount := countNodes(t, "Author")
if authorCount != 2 {
t.Errorf("Expected 2 Author nodes, got %d", authorCount)
}
// Run migration
err := migrateAuthorToNostrUser(ctx, testDB)
if err != nil {
t.Fatalf("Migration failed: %v", err)
}
// Verify NostrUser nodes were created
nostrUserCount := countNodes(t, "NostrUser")
if nostrUserCount != 2 {
t.Errorf("Expected 2 NostrUser nodes after migration, got %d", nostrUserCount)
}
// Verify Author nodes were deleted (they should have no relationships after migration)
authorCountAfter := countNodes(t, "Author")
if authorCountAfter != 0 {
t.Errorf("Expected 0 Author nodes after migration, got %d", authorCountAfter)
}
}

View File

@@ -55,43 +55,71 @@ func (n *N) buildCypherQuery(f *filter.F, includeDeleteEvents bool) (string, map
matchClause := "MATCH (e:Event)"
// IDs filter - uses exact match or prefix matching
if len(f.Ids.T) > 0 {
idConditions := make([]string, len(f.Ids.T))
// Note: IDs can be either binary (32 bytes) or hex strings (64 chars)
// We need to normalize to lowercase hex for consistent Neo4j matching
if f.Ids != nil && len(f.Ids.T) > 0 {
idConditions := make([]string, 0, len(f.Ids.T))
for i, id := range f.Ids.T {
if len(id) == 0 {
continue // Skip empty IDs
}
paramName := fmt.Sprintf("id_%d", i)
hexID := hex.Enc(id)
// Normalize to lowercase hex using our utility function
// This handles both binary-encoded IDs and hex string IDs (including uppercase)
hexID := NormalizePubkeyHex(id)
if hexID == "" {
continue
}
// Handle prefix matching for partial IDs
if len(id) < 32 { // Full event ID is 32 bytes (64 hex chars)
idConditions[i] = fmt.Sprintf("e.id STARTS WITH $%s", paramName)
// After normalization, check hex length (should be 64 for full ID)
if len(hexID) < 64 {
idConditions = append(idConditions, fmt.Sprintf("e.id STARTS WITH $%s", paramName))
} else {
idConditions[i] = fmt.Sprintf("e.id = $%s", paramName)
idConditions = append(idConditions, fmt.Sprintf("e.id = $%s", paramName))
}
params[paramName] = hexID
}
whereClauses = append(whereClauses, "("+strings.Join(idConditions, " OR ")+")")
if len(idConditions) > 0 {
whereClauses = append(whereClauses, "("+strings.Join(idConditions, " OR ")+")")
}
}
// Authors filter - supports prefix matching for partial pubkeys
if len(f.Authors.T) > 0 {
authorConditions := make([]string, len(f.Authors.T))
// Note: Authors can be either binary (32 bytes) or hex strings (64 chars)
// We need to normalize to lowercase hex for consistent Neo4j matching
if f.Authors != nil && len(f.Authors.T) > 0 {
authorConditions := make([]string, 0, len(f.Authors.T))
for i, author := range f.Authors.T {
if len(author) == 0 {
continue // Skip empty authors
}
paramName := fmt.Sprintf("author_%d", i)
hexAuthor := hex.Enc(author)
// Normalize to lowercase hex using our utility function
// This handles both binary-encoded pubkeys and hex string pubkeys (including uppercase)
hexAuthor := NormalizePubkeyHex(author)
if hexAuthor == "" {
continue
}
// Handle prefix matching for partial pubkeys
if len(author) < 32 { // Full pubkey is 32 bytes (64 hex chars)
authorConditions[i] = fmt.Sprintf("e.pubkey STARTS WITH $%s", paramName)
// After normalization, check hex length (should be 64 for full pubkey)
if len(hexAuthor) < 64 {
authorConditions = append(authorConditions, fmt.Sprintf("e.pubkey STARTS WITH $%s", paramName))
} else {
authorConditions[i] = fmt.Sprintf("e.pubkey = $%s", paramName)
authorConditions = append(authorConditions, fmt.Sprintf("e.pubkey = $%s", paramName))
}
params[paramName] = hexAuthor
}
whereClauses = append(whereClauses, "("+strings.Join(authorConditions, " OR ")+")")
if len(authorConditions) > 0 {
whereClauses = append(whereClauses, "("+strings.Join(authorConditions, " OR ")+")")
}
}
// Kinds filter - matches event types
if len(f.Kinds.K) > 0 {
if f.Kinds != nil && len(f.Kinds.K) > 0 {
kinds := make([]int64, len(f.Kinds.K))
for i, k := range f.Kinds.K {
kinds[i] = int64(k.K)

View File

@@ -0,0 +1,314 @@
package neo4j
import (
"context"
"testing"
"git.mleku.dev/mleku/nostr/encoders/filter"
"git.mleku.dev/mleku/nostr/encoders/kind"
"git.mleku.dev/mleku/nostr/encoders/tag"
"git.mleku.dev/mleku/nostr/encoders/timestamp"
)
// Valid test pubkeys and event IDs (64-character lowercase hex)
const (
validPubkey1 = "0000000000000000000000000000000000000000000000000000000000000001"
validPubkey2 = "0000000000000000000000000000000000000000000000000000000000000002"
validPubkey3 = "0000000000000000000000000000000000000000000000000000000000000003"
validEventID1 = "1111111111111111111111111111111111111111111111111111111111111111"
validEventID2 = "2222222222222222222222222222222222222222222222222222222222222222"
validEventID3 = "3333333333333333333333333333333333333333333333333333333333333333"
)
// TestQueryEventsWithNilFilter tests that QueryEvents handles nil filter fields gracefully
// This test covers the nil pointer fix in query-events.go
func TestQueryEventsWithNilFilter(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
// Setup some test events
setupTestEvent(t, validEventID1, validPubkey1, 1, "[]")
setupTestEvent(t, validEventID2, validPubkey2, 1, "[]")
ctx := context.Background()
// Test 1: Completely empty filter (all nil fields)
t.Run("EmptyFilter", func(t *testing.T) {
f := &filter.F{}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents with empty filter should not panic: %v", err)
}
if len(events) == 0 {
t.Error("Expected to find events with empty filter")
}
})
// Test 2: Filter with nil Ids
t.Run("NilIds", func(t *testing.T) {
f := &filter.F{
Ids: nil, // Explicitly nil
}
_, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents with nil Ids should not panic: %v", err)
}
})
// Test 3: Filter with nil Authors
t.Run("NilAuthors", func(t *testing.T) {
f := &filter.F{
Authors: nil, // Explicitly nil
}
_, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents with nil Authors should not panic: %v", err)
}
})
// Test 4: Filter with nil Kinds
t.Run("NilKinds", func(t *testing.T) {
f := &filter.F{
Kinds: nil, // Explicitly nil
}
_, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents with nil Kinds should not panic: %v", err)
}
})
// Test 5: Filter with empty Ids slice
t.Run("EmptyIds", func(t *testing.T) {
f := &filter.F{
Ids: &tag.S{T: [][]byte{}},
}
_, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents with empty Ids should not panic: %v", err)
}
})
// Test 6: Filter with empty Authors slice
t.Run("EmptyAuthors", func(t *testing.T) {
f := &filter.F{
Authors: &tag.S{T: [][]byte{}},
}
_, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents with empty Authors should not panic: %v", err)
}
})
// Test 7: Filter with empty Kinds slice
t.Run("EmptyKinds", func(t *testing.T) {
f := &filter.F{
Kinds: &kind.S{K: []*kind.T{}},
}
_, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents with empty Kinds should not panic: %v", err)
}
})
}
// TestQueryEventsWithValidFilters tests that QueryEvents works correctly with valid filters
func TestQueryEventsWithValidFilters(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
// Setup test events
setupTestEvent(t, validEventID1, validPubkey1, 1, "[]")
setupTestEvent(t, validEventID2, validPubkey2, 3, "[]")
setupTestEvent(t, validEventID3, validPubkey1, 1, "[]")
ctx := context.Background()
// Test 1: Filter by ID
t.Run("FilterByID", func(t *testing.T) {
f := &filter.F{
Ids: tag.NewFromBytesSlice([]byte(validEventID1)),
}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents failed: %v", err)
}
if len(events) != 1 {
t.Errorf("Expected 1 event, got %d", len(events))
}
})
// Test 2: Filter by Author
t.Run("FilterByAuthor", func(t *testing.T) {
f := &filter.F{
Authors: tag.NewFromBytesSlice([]byte(validPubkey1)),
}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents failed: %v", err)
}
if len(events) != 2 {
t.Errorf("Expected 2 events from pubkey1, got %d", len(events))
}
})
// Test 3: Filter by Kind
t.Run("FilterByKind", func(t *testing.T) {
f := &filter.F{
Kinds: kind.NewS(kind.New(1)),
}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents failed: %v", err)
}
if len(events) != 2 {
t.Errorf("Expected 2 kind-1 events, got %d", len(events))
}
})
// Test 4: Combined filters (kind + author)
t.Run("FilterByKindAndAuthor", func(t *testing.T) {
f := &filter.F{
Kinds: kind.NewS(kind.New(1)),
Authors: tag.NewFromBytesSlice([]byte(validPubkey1)),
}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents failed: %v", err)
}
if len(events) != 2 {
t.Errorf("Expected 2 kind-1 events from pubkey1, got %d", len(events))
}
})
// Test 5: Filter with limit
t.Run("FilterWithLimit", func(t *testing.T) {
limit := 1
f := &filter.F{
Kinds: kind.NewS(kind.New(1)),
Limit: &limit,
}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents failed: %v", err)
}
if len(events) != 1 {
t.Errorf("Expected 1 event due to limit, got %d", len(events))
}
})
}
// TestBuildCypherQueryWithNilFields tests the buildCypherQuery function with nil fields
func TestBuildCypherQueryWithNilFields(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Test that buildCypherQuery doesn't panic with nil fields
t.Run("AllNilFields", func(t *testing.T) {
f := &filter.F{
Ids: nil,
Authors: nil,
Kinds: nil,
Since: nil,
Until: nil,
Tags: nil,
Limit: nil,
}
cypher, params := testDB.buildCypherQuery(f, false)
if cypher == "" {
t.Error("Expected non-empty Cypher query")
}
if params == nil {
t.Error("Expected non-nil params map")
}
})
// Test with empty slices
t.Run("EmptySlices", func(t *testing.T) {
f := &filter.F{
Ids: &tag.S{T: [][]byte{}},
Authors: &tag.S{T: [][]byte{}},
Kinds: &kind.S{K: []*kind.T{}},
}
cypher, params := testDB.buildCypherQuery(f, false)
if cypher == "" {
t.Error("Expected non-empty Cypher query")
}
if params == nil {
t.Error("Expected non-nil params map")
}
})
// Test with time filters
t.Run("TimeFilters", func(t *testing.T) {
since := timestamp.Now()
until := timestamp.Now()
f := &filter.F{
Since: &since,
Until: &until,
}
cypher, params := testDB.buildCypherQuery(f, false)
if _, ok := params["since"]; !ok {
t.Error("Expected 'since' param")
}
if _, ok := params["until"]; !ok {
t.Error("Expected 'until' param")
}
_ = cypher
})
}
// TestQueryEventsUppercaseHexNormalization tests that uppercase hex in filters is normalized
func TestQueryEventsUppercaseHexNormalization(t *testing.T) {
if testDB == nil {
t.Skip("Neo4j not available")
}
// Clean up before test
cleanTestDatabase()
// Setup test event with lowercase pubkey (as Neo4j stores)
lowercasePubkey := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
lowercaseEventID := "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"
setupTestEvent(t, lowercaseEventID, lowercasePubkey, 1, "[]")
ctx := context.Background()
// Test query with uppercase pubkey - should be normalized and still match
t.Run("UppercaseAuthor", func(t *testing.T) {
uppercasePubkey := "ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789"
f := &filter.F{
Authors: tag.NewFromBytesSlice([]byte(uppercasePubkey)),
}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents failed: %v", err)
}
if len(events) != 1 {
t.Errorf("Expected to find 1 event with uppercase pubkey filter, got %d", len(events))
}
})
// Test query with uppercase event ID - should be normalized and still match
t.Run("UppercaseEventID", func(t *testing.T) {
uppercaseEventID := "FEDCBA9876543210FEDCBA9876543210FEDCBA9876543210FEDCBA9876543210"
f := &filter.F{
Ids: tag.NewFromBytesSlice([]byte(uppercaseEventID)),
}
events, err := testDB.QueryEvents(ctx, f)
if err != nil {
t.Fatalf("QueryEvents failed: %v", err)
}
if len(events) != 1 {
t.Errorf("Expected to find 1 event with uppercase ID filter, got %d", len(events))
}
})
}

50
pkg/neo4j/run-tests.sh Executable file
View File

@@ -0,0 +1,50 @@
#!/bin/bash
# Run Neo4j integration tests with Docker
# Usage: ./run-tests.sh
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
echo "Starting Neo4j test database..."
docker compose up -d
echo "Waiting for Neo4j to be ready..."
for i in {1..30}; do
if docker compose exec -T neo4j-test cypher-shell -u neo4j -p testpassword "RETURN 1" > /dev/null 2>&1; then
echo "Neo4j is ready!"
break
fi
if [ $i -eq 30 ]; then
echo "Timeout waiting for Neo4j"
docker compose logs
docker compose down
exit 1
fi
echo "Waiting... ($i/30)"
sleep 2
done
echo ""
echo "Running tests..."
echo "================="
# Set environment variables for tests
export NEO4J_TEST_URI="bolt://localhost:7687"
export NEO4J_TEST_USER="neo4j"
export NEO4J_TEST_PASSWORD="testpassword"
# Run tests with verbose output
cd ../..
CGO_ENABLED=0 go test -v ./pkg/neo4j/... -count=1
TEST_EXIT_CODE=$?
cd "$SCRIPT_DIR"
echo ""
echo "================="
echo "Stopping Neo4j test database..."
docker compose down
exit $TEST_EXIT_CODE

View File

@@ -158,6 +158,14 @@ CREATE (e)-[:AUTHORED_BY]->(a)
// This is required because Cypher doesn't allow MATCH after CREATE without WITH
needsWithClause := true
// Collect all e-tags, p-tags, and other tags first so we can generate proper Cypher
// Neo4j requires WITH clauses between certain clause types (FOREACH -> MATCH/MERGE)
type tagInfo struct {
tagType string
value string
}
var eTags, pTags, otherTags []tagInfo
// Only process tags if they exist
if ev.Tags != nil {
for _, tagItem := range *ev.Tags {
@@ -168,30 +176,39 @@ CREATE (e)-[:AUTHORED_BY]->(a)
tagType := string(tagItem.T[0])
switch tagType {
case "e": // Event reference - creates REFERENCES relationship
// Use ExtractETagValue to handle binary encoding and normalize to lowercase hex
case "e": // Event reference
tagValue := ExtractETagValue(tagItem)
if tagValue == "" {
continue // Skip invalid e-tags
if tagValue != "" {
eTags = append(eTags, tagInfo{"e", tagValue})
}
case "p": // Pubkey mention
tagValue := ExtractPTagValue(tagItem)
if tagValue != "" {
pTags = append(pTags, tagInfo{"p", tagValue})
}
default: // Other tags
tagValue := string(tagItem.T[1])
otherTags = append(otherTags, tagInfo{tagType, tagValue})
}
}
}
// Create reference to another event (if it exists)
paramName := fmt.Sprintf("eTag_%d", eTagIndex)
params[paramName] = tagValue
// Generate Cypher for e-tags (OPTIONAL MATCH + FOREACH pattern)
// These need WITH clause before first one, and WITH after all FOREACHes
for i, tag := range eTags {
paramName := fmt.Sprintf("eTag_%d", eTagIndex)
params[paramName] = tag.value
// Add WITH clause before first OPTIONAL MATCH only
// This is required because Cypher doesn't allow MATCH after CREATE without WITH.
// However, you CAN chain multiple OPTIONAL MATCH + FOREACH pairs without
// additional WITH clauses between them - Cypher allows OPTIONAL MATCH after FOREACH.
if needsWithClause {
cypher += `
// Add WITH clause before first OPTIONAL MATCH only
if needsWithClause {
cypher += `
// Carry forward event and author nodes for tag processing
WITH e, a
`
needsWithClause = false
}
needsWithClause = false
}
cypher += fmt.Sprintf(`
cypher += fmt.Sprintf(`
// Reference to event (e-tag)
OPTIONAL MATCH (ref%d:Event {id: $%s})
FOREACH (ignoreMe IN CASE WHEN ref%d IS NOT NULL THEN [1] ELSE [] END |
@@ -199,47 +216,64 @@ FOREACH (ignoreMe IN CASE WHEN ref%d IS NOT NULL THEN [1] ELSE [] END |
)
`, eTagIndex, paramName, eTagIndex, eTagIndex)
eTagIndex++
eTagIndex++
case "p": // Pubkey mention - creates MENTIONS relationship
// Use ExtractPTagValue to handle binary encoding and normalize to lowercase hex
tagValue := ExtractPTagValue(tagItem)
if tagValue == "" {
continue // Skip invalid p-tags
}
// After the last e-tag FOREACH, add WITH clause if there are p-tags or other tags
if i == len(eTags)-1 && (len(pTags) > 0 || len(otherTags) > 0) {
cypher += `
// Required WITH after FOREACH before MERGE/MATCH
WITH e, a
`
}
}
// Create mention to another NostrUser
paramName := fmt.Sprintf("pTag_%d", pTagIndex)
params[paramName] = tagValue
// Generate Cypher for p-tags (MERGE pattern)
for _, tag := range pTags {
paramName := fmt.Sprintf("pTag_%d", pTagIndex)
params[paramName] = tag.value
cypher += fmt.Sprintf(`
// If no e-tags were processed, we still need the initial WITH
if needsWithClause {
cypher += `
// Carry forward event and author nodes for tag processing
WITH e, a
`
needsWithClause = false
}
cypher += fmt.Sprintf(`
// Mention of NostrUser (p-tag)
MERGE (mentioned%d:NostrUser {pubkey: $%s})
ON CREATE SET mentioned%d.created_at = timestamp()
CREATE (e)-[:MENTIONS]->(mentioned%d)
`, pTagIndex, paramName, pTagIndex, pTagIndex)
pTagIndex++
pTagIndex++
}
default: // Other tags - creates Tag nodes and TAGGED_WITH relationships
// For non-e/p tags, use direct string conversion (no binary encoding)
tagValue := string(tagItem.T[1])
// Generate Cypher for other tags (MERGE pattern)
for _, tag := range otherTags {
typeParam := fmt.Sprintf("tagType_%d", tagNodeIndex)
valueParam := fmt.Sprintf("tagValue_%d", tagNodeIndex)
params[typeParam] = tag.tagType
params[valueParam] = tag.value
// Create tag node and relationship
typeParam := fmt.Sprintf("tagType_%d", tagNodeIndex)
valueParam := fmt.Sprintf("tagValue_%d", tagNodeIndex)
params[typeParam] = tagType
params[valueParam] = tagValue
// If no e-tags or p-tags were processed, we still need the initial WITH
if needsWithClause {
cypher += `
// Carry forward event and author nodes for tag processing
WITH e, a
`
needsWithClause = false
}
cypher += fmt.Sprintf(`
cypher += fmt.Sprintf(`
// Generic tag relationship
MERGE (tag%d:Tag {type: $%s, value: $%s})
CREATE (e)-[:TAGGED_WITH]->(tag%d)
`, tagNodeIndex, typeParam, valueParam, tagNodeIndex)
tagNodeIndex++
}
}
tagNodeIndex++
}
// Return the created event

View File

@@ -1,15 +1,246 @@
package neo4j
import (
"context"
"os"
"testing"
"time"
"next.orly.dev/pkg/database"
)
// skipIfNeo4jNotAvailable skips the test if Neo4j is not available
func skipIfNeo4jNotAvailable(t *testing.T) {
// Check if Neo4j connection details are provided
uri := os.Getenv("ORLY_NEO4J_URI")
if uri == "" {
t.Skip("Neo4j not available (set ORLY_NEO4J_URI to enable tests)")
// testDB is the shared database instance for tests
var testDB *N
// TestMain sets up and tears down the test database
func TestMain(m *testing.M) {
// Skip integration tests if NEO4J_TEST_URI is not set
neo4jURI := os.Getenv("NEO4J_TEST_URI")
if neo4jURI == "" {
neo4jURI = "bolt://localhost:7687"
}
neo4jUser := os.Getenv("NEO4J_TEST_USER")
if neo4jUser == "" {
neo4jUser = "neo4j"
}
neo4jPassword := os.Getenv("NEO4J_TEST_PASSWORD")
if neo4jPassword == "" {
neo4jPassword = "testpassword"
}
// Try to connect to Neo4j
ctx, cancel := context.WithCancel(context.Background())
cfg := &database.DatabaseConfig{
DataDir: os.TempDir(),
Neo4jURI: neo4jURI,
Neo4jUser: neo4jUser,
Neo4jPassword: neo4jPassword,
}
var err error
testDB, err = NewWithConfig(ctx, cancel, cfg)
if err != nil {
// If Neo4j is not available, skip integration tests
os.Stderr.WriteString("Neo4j not available, skipping integration tests: " + err.Error() + "\n")
os.Stderr.WriteString("Start Neo4j with: docker compose -f pkg/neo4j/docker-compose.yaml up -d\n")
os.Exit(0)
}
// Wait for database to be ready
select {
case <-testDB.Ready():
// Database is ready
case <-time.After(30 * time.Second):
os.Stderr.WriteString("Timeout waiting for Neo4j to be ready\n")
os.Exit(1)
}
// Clean database before running tests
cleanTestDatabase()
// Run tests
code := m.Run()
// Clean up
cleanTestDatabase()
testDB.Close()
cancel()
os.Exit(code)
}
// cleanTestDatabase removes all nodes and relationships
func cleanTestDatabase() {
ctx := context.Background()
// Delete all nodes and relationships
_, _ = testDB.ExecuteWrite(ctx, "MATCH (n) DETACH DELETE n", nil)
// Clear migration markers so migrations can run fresh
_, _ = testDB.ExecuteWrite(ctx, "MATCH (m:Migration) DELETE m", nil)
}
// setupTestEvent creates a test event directly in Neo4j for testing queries
func setupTestEvent(t *testing.T, eventID, pubkey string, kind int64, tags string) {
t.Helper()
ctx := context.Background()
cypher := `
MERGE (a:NostrUser {pubkey: $pubkey})
CREATE (e:Event {
id: $eventId,
serial: $serial,
kind: $kind,
created_at: $createdAt,
content: $content,
sig: $sig,
pubkey: $pubkey,
tags: $tags,
expiration: 0
})
CREATE (e)-[:AUTHORED_BY]->(a)
`
params := map[string]any{
"eventId": eventID,
"serial": time.Now().UnixNano(),
"kind": kind,
"createdAt": time.Now().Unix(),
"content": "test content",
"sig": "0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000",
"pubkey": pubkey,
"tags": tags,
}
_, err := testDB.ExecuteWrite(ctx, cypher, params)
if err != nil {
t.Fatalf("Failed to setup test event: %v", err)
}
}
// setupInvalidNostrUser creates a NostrUser with an invalid (binary) pubkey for testing migrations
func setupInvalidNostrUser(t *testing.T, invalidPubkey string) {
t.Helper()
ctx := context.Background()
cypher := `CREATE (u:NostrUser {pubkey: $pubkey, created_at: timestamp()})`
params := map[string]any{"pubkey": invalidPubkey}
_, err := testDB.ExecuteWrite(ctx, cypher, params)
if err != nil {
t.Fatalf("Failed to setup invalid NostrUser: %v", err)
}
}
// setupInvalidEvent creates an Event with an invalid pubkey/ID for testing migrations
func setupInvalidEvent(t *testing.T, invalidID, invalidPubkey string) {
t.Helper()
ctx := context.Background()
cypher := `
CREATE (e:Event {
id: $id,
pubkey: $pubkey,
kind: 1,
created_at: timestamp(),
content: 'test',
sig: 'invalid',
tags: '[]',
serial: $serial,
expiration: 0
})
`
params := map[string]any{
"id": invalidID,
"pubkey": invalidPubkey,
"serial": time.Now().UnixNano(),
}
_, err := testDB.ExecuteWrite(ctx, cypher, params)
if err != nil {
t.Fatalf("Failed to setup invalid Event: %v", err)
}
}
// setupInvalidTag creates a Tag node with invalid value for testing migrations
func setupInvalidTag(t *testing.T, tagType string, invalidValue string) {
t.Helper()
ctx := context.Background()
cypher := `CREATE (tag:Tag {type: $type, value: $value})`
params := map[string]any{
"type": tagType,
"value": invalidValue,
}
_, err := testDB.ExecuteWrite(ctx, cypher, params)
if err != nil {
t.Fatalf("Failed to setup invalid Tag: %v", err)
}
}
// countNodes counts nodes with a given label
func countNodes(t *testing.T, label string) int64 {
t.Helper()
ctx := context.Background()
cypher := "MATCH (n:" + label + ") RETURN count(n) AS count"
result, err := testDB.ExecuteRead(ctx, cypher, nil)
if err != nil {
t.Fatalf("Failed to count nodes: %v", err)
}
if result.Next(ctx) {
if count, ok := result.Record().Values[0].(int64); ok {
return count
}
}
return 0
}
// countInvalidNostrUsers counts NostrUser nodes with invalid pubkeys
func countInvalidNostrUsers(t *testing.T) int64 {
t.Helper()
ctx := context.Background()
cypher := `
MATCH (u:NostrUser)
WHERE size(u.pubkey) <> 64
OR NOT u.pubkey =~ '^[0-9a-f]{64}$'
RETURN count(u) AS count
`
result, err := testDB.ExecuteRead(ctx, cypher, nil)
if err != nil {
t.Fatalf("Failed to count invalid NostrUsers: %v", err)
}
if result.Next(ctx) {
if count, ok := result.Record().Values[0].(int64); ok {
return count
}
}
return 0
}
// countInvalidTags counts Tag nodes (e/p type) with invalid values
func countInvalidTags(t *testing.T) int64 {
t.Helper()
ctx := context.Background()
cypher := `
MATCH (t:Tag)
WHERE t.type IN ['e', 'p']
AND (size(t.value) <> 64 OR NOT t.value =~ '^[0-9a-f]{64}$')
RETURN count(t) AS count
`
result, err := testDB.ExecuteRead(ctx, cypher, nil)
if err != nil {
t.Fatalf("Failed to count invalid Tags: %v", err)
}
if result.Next(ctx) {
if count, ok := result.Record().Values[0].(int64); ok {
return count
}
}
return 0
}

View File

@@ -1 +1 @@
v0.34.0
v0.34.2