This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
|
||||
A comprehensive benchmarking system for testing and comparing the performance of multiple Nostr relay implementations, including:
|
||||
|
||||
- **next.orly.dev** (this repository) - Badger, DGraph, and Neo4j backend variants
|
||||
- **next.orly.dev** (this repository) - Badger and Neo4j backend variants
|
||||
- **Khatru** - SQLite and Badger variants
|
||||
- **Relayer** - Basic example implementation
|
||||
- **Strfry** - C++ LMDB-based relay
|
||||
@@ -94,10 +94,7 @@ ls reports/run_YYYYMMDD_HHMMSS/
|
||||
| Service | Port | Description |
|
||||
| ------------------ | ---- | ----------------------------------------- |
|
||||
| next-orly-badger | 8001 | This repository's Badger relay |
|
||||
| next-orly-dgraph | 8007 | This repository's DGraph relay |
|
||||
| next-orly-neo4j | 8008 | This repository's Neo4j relay |
|
||||
| dgraph-zero | 5080 | DGraph cluster coordinator |
|
||||
| dgraph-alpha | 9080 | DGraph data node |
|
||||
| neo4j | 7474/7687 | Neo4j graph database |
|
||||
| khatru-sqlite | 8002 | Khatru with SQLite backend |
|
||||
| khatru-badger | 8003 | Khatru with Badger backend |
|
||||
@@ -180,7 +177,7 @@ go build -o benchmark main.go
|
||||
|
||||
## Database Backend Comparison
|
||||
|
||||
The benchmark suite includes **next.orly.dev** with three different database backends to compare architectural approaches:
|
||||
The benchmark suite includes **next.orly.dev** with two different database backends to compare architectural approaches:
|
||||
|
||||
### Badger Backend (next-orly-badger)
|
||||
- **Type**: Embedded key-value store
|
||||
@@ -192,16 +189,6 @@ The benchmark suite includes **next.orly.dev** with three different database bac
|
||||
- Simpler deployment
|
||||
- Limited to single-node scaling
|
||||
|
||||
### DGraph Backend (next-orly-dgraph)
|
||||
- **Type**: Distributed graph database
|
||||
- **Architecture**: Client-server with dgraph-zero (coordinator) and dgraph-alpha (data node)
|
||||
- **Best for**: Distributed deployments, horizontal scaling
|
||||
- **Characteristics**:
|
||||
- Network overhead from gRPC communication
|
||||
- Supports multi-node clustering
|
||||
- Built-in replication and sharding
|
||||
- More complex deployment
|
||||
|
||||
### Neo4j Backend (next-orly-neo4j)
|
||||
- **Type**: Native graph database
|
||||
- **Architecture**: Client-server with Neo4j Community Edition
|
||||
@@ -218,10 +205,10 @@ The benchmark suite includes **next.orly.dev** with three different database bac
|
||||
### Comparing the Backends
|
||||
|
||||
The benchmark results will show:
|
||||
- **Latency differences**: Embedded vs. distributed overhead, graph traversal efficiency
|
||||
- **Throughput trade-offs**: Single-process optimization vs. distributed scalability vs. graph query optimization
|
||||
- **Latency differences**: Embedded vs. client-server overhead, graph traversal efficiency
|
||||
- **Throughput trade-offs**: Single-process optimization vs. graph query optimization
|
||||
- **Resource usage**: Memory and CPU patterns for different architectures
|
||||
- **Query performance**: Graph queries (Neo4j) vs. key-value lookups (Badger) vs. distributed queries (DGraph)
|
||||
- **Query performance**: Graph queries (Neo4j) vs. key-value lookups (Badger)
|
||||
|
||||
This comparison helps determine which backend is appropriate for different deployment scenarios and workload patterns.
|
||||
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"next.orly.dev/pkg/database"
|
||||
_ "next.orly.dev/pkg/dgraph" // Import to register dgraph factory
|
||||
)
|
||||
|
||||
// DgraphBenchmark wraps a Benchmark with dgraph-specific setup
|
||||
type DgraphBenchmark struct {
|
||||
config *BenchmarkConfig
|
||||
docker *DgraphDocker
|
||||
database database.Database
|
||||
bench *BenchmarkAdapter
|
||||
}
|
||||
|
||||
// NewDgraphBenchmark creates a new dgraph benchmark instance
|
||||
func NewDgraphBenchmark(config *BenchmarkConfig) (*DgraphBenchmark, error) {
|
||||
// Create Docker manager
|
||||
docker := NewDgraphDocker()
|
||||
|
||||
// Start dgraph containers
|
||||
ctx := context.Background()
|
||||
if err := docker.Start(ctx); err != nil {
|
||||
return nil, fmt.Errorf("failed to start dgraph: %w", err)
|
||||
}
|
||||
|
||||
// Set environment variable for dgraph connection
|
||||
os.Setenv("ORLY_DGRAPH_URL", docker.GetGRPCEndpoint())
|
||||
|
||||
// Create database instance using dgraph backend
|
||||
cancel := func() {}
|
||||
db, err := database.NewDatabase(ctx, cancel, "dgraph", config.DataDir, "warn")
|
||||
if err != nil {
|
||||
docker.Stop()
|
||||
return nil, fmt.Errorf("failed to create dgraph database: %w", err)
|
||||
}
|
||||
|
||||
// Wait for database to be ready
|
||||
fmt.Println("Waiting for dgraph database to be ready...")
|
||||
select {
|
||||
case <-db.Ready():
|
||||
fmt.Println("Dgraph database is ready")
|
||||
case <-time.After(30 * time.Second):
|
||||
db.Close()
|
||||
docker.Stop()
|
||||
return nil, fmt.Errorf("dgraph database failed to become ready")
|
||||
}
|
||||
|
||||
// Create adapter to use Database interface with Benchmark
|
||||
adapter := NewBenchmarkAdapter(config, db)
|
||||
|
||||
dgraphBench := &DgraphBenchmark{
|
||||
config: config,
|
||||
docker: docker,
|
||||
database: db,
|
||||
bench: adapter,
|
||||
}
|
||||
|
||||
return dgraphBench, nil
|
||||
}
|
||||
|
||||
// Close closes the dgraph benchmark and stops Docker containers
|
||||
func (dgb *DgraphBenchmark) Close() {
|
||||
fmt.Println("Closing dgraph benchmark...")
|
||||
|
||||
if dgb.database != nil {
|
||||
dgb.database.Close()
|
||||
}
|
||||
|
||||
if dgb.docker != nil {
|
||||
if err := dgb.docker.Stop(); err != nil {
|
||||
log.Printf("Error stopping dgraph Docker: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RunSuite runs the benchmark suite on dgraph
|
||||
func (dgb *DgraphBenchmark) RunSuite() {
|
||||
fmt.Println("\n╔════════════════════════════════════════════════════════╗")
|
||||
fmt.Println("║ DGRAPH BACKEND BENCHMARK SUITE ║")
|
||||
fmt.Println("╚════════════════════════════════════════════════════════╝")
|
||||
|
||||
// Run only one round for dgraph to keep benchmark time reasonable
|
||||
fmt.Printf("\n=== Starting dgraph benchmark ===\n")
|
||||
|
||||
fmt.Printf("RunPeakThroughputTest (dgraph)..\n")
|
||||
dgb.bench.RunPeakThroughputTest()
|
||||
fmt.Println("Wiping database between tests...")
|
||||
dgb.database.Wipe()
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
fmt.Printf("RunBurstPatternTest (dgraph)..\n")
|
||||
dgb.bench.RunBurstPatternTest()
|
||||
fmt.Println("Wiping database between tests...")
|
||||
dgb.database.Wipe()
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
fmt.Printf("RunMixedReadWriteTest (dgraph)..\n")
|
||||
dgb.bench.RunMixedReadWriteTest()
|
||||
fmt.Println("Wiping database between tests...")
|
||||
dgb.database.Wipe()
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
fmt.Printf("RunQueryTest (dgraph)..\n")
|
||||
dgb.bench.RunQueryTest()
|
||||
fmt.Println("Wiping database between tests...")
|
||||
dgb.database.Wipe()
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
fmt.Printf("RunConcurrentQueryStoreTest (dgraph)..\n")
|
||||
dgb.bench.RunConcurrentQueryStoreTest()
|
||||
|
||||
fmt.Printf("\n=== Dgraph benchmark completed ===\n\n")
|
||||
}
|
||||
|
||||
// GenerateReport generates the benchmark report
|
||||
func (dgb *DgraphBenchmark) GenerateReport() {
|
||||
dgb.bench.GenerateReport()
|
||||
}
|
||||
|
||||
// GenerateAsciidocReport generates asciidoc format report
|
||||
func (dgb *DgraphBenchmark) GenerateAsciidocReport() {
|
||||
dgb.bench.GenerateAsciidocReport()
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DgraphDocker manages a dgraph instance via Docker Compose
|
||||
type DgraphDocker struct {
|
||||
composeFile string
|
||||
projectName string
|
||||
running bool
|
||||
}
|
||||
|
||||
// NewDgraphDocker creates a new dgraph Docker manager
|
||||
func NewDgraphDocker() *DgraphDocker {
|
||||
// Try to find the docker-compose file in the current directory first
|
||||
composeFile := "docker-compose-dgraph.yml"
|
||||
|
||||
// If not found, try the cmd/benchmark directory (for running from project root)
|
||||
if _, err := os.Stat(composeFile); os.IsNotExist(err) {
|
||||
composeFile = filepath.Join("cmd", "benchmark", "docker-compose-dgraph.yml")
|
||||
}
|
||||
|
||||
return &DgraphDocker{
|
||||
composeFile: composeFile,
|
||||
projectName: "orly-benchmark-dgraph",
|
||||
running: false,
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts the dgraph Docker containers
|
||||
func (d *DgraphDocker) Start(ctx context.Context) error {
|
||||
fmt.Println("Starting dgraph Docker containers...")
|
||||
|
||||
// Stop any existing containers first
|
||||
d.Stop()
|
||||
|
||||
// Start containers
|
||||
cmd := exec.CommandContext(
|
||||
ctx,
|
||||
"docker-compose",
|
||||
"-f", d.composeFile,
|
||||
"-p", d.projectName,
|
||||
"up", "-d",
|
||||
)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to start dgraph containers: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Waiting for dgraph to be healthy...")
|
||||
|
||||
// Wait for health checks to pass
|
||||
if err := d.waitForHealthy(ctx, 60*time.Second); err != nil {
|
||||
d.Stop() // Clean up on failure
|
||||
return err
|
||||
}
|
||||
|
||||
d.running = true
|
||||
fmt.Println("Dgraph is ready!")
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForHealthy waits for dgraph to become healthy
|
||||
func (d *DgraphDocker) waitForHealthy(ctx context.Context, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
// Check if alpha is healthy by checking docker health status
|
||||
cmd := exec.CommandContext(
|
||||
ctx,
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format={{.State.Health.Status}}",
|
||||
"orly-benchmark-dgraph-alpha",
|
||||
)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err == nil && string(output) == "healthy\n" {
|
||||
// Additional short wait to ensure full readiness
|
||||
time.Sleep(2 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(2 * time.Second):
|
||||
// Continue waiting
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("dgraph failed to become healthy within %v", timeout)
|
||||
}
|
||||
|
||||
// Stop stops and removes the dgraph Docker containers
|
||||
func (d *DgraphDocker) Stop() error {
|
||||
if !d.running {
|
||||
// Try to stop anyway in case of untracked state
|
||||
cmd := exec.Command(
|
||||
"docker-compose",
|
||||
"-f", d.composeFile,
|
||||
"-p", d.projectName,
|
||||
"down", "-v",
|
||||
)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
_ = cmd.Run() // Ignore errors
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Println("Stopping dgraph Docker containers...")
|
||||
|
||||
cmd := exec.Command(
|
||||
"docker-compose",
|
||||
"-f", d.composeFile,
|
||||
"-p", d.projectName,
|
||||
"down", "-v",
|
||||
)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to stop dgraph containers: %w", err)
|
||||
}
|
||||
|
||||
d.running = false
|
||||
fmt.Println("Dgraph containers stopped")
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetGRPCEndpoint returns the dgraph gRPC endpoint
|
||||
func (d *DgraphDocker) GetGRPCEndpoint() string {
|
||||
return "localhost:9080"
|
||||
}
|
||||
|
||||
// IsRunning returns whether dgraph is running
|
||||
func (d *DgraphDocker) IsRunning() bool {
|
||||
return d.running
|
||||
}
|
||||
|
||||
// Logs returns the logs from dgraph containers
|
||||
func (d *DgraphDocker) Logs() error {
|
||||
cmd := exec.Command(
|
||||
"docker-compose",
|
||||
"-f", d.composeFile,
|
||||
"-p", d.projectName,
|
||||
"logs",
|
||||
)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
dgraph-zero:
|
||||
image: dgraph/dgraph:v23.1.0
|
||||
container_name: orly-benchmark-dgraph-zero
|
||||
working_dir: /data/zero
|
||||
ports:
|
||||
- "5080:5080"
|
||||
- "6080:6080"
|
||||
command: dgraph zero --my=dgraph-zero:5080
|
||||
networks:
|
||||
- orly-benchmark
|
||||
healthcheck:
|
||||
test: ["CMD", "sh", "-c", "dgraph version || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
|
||||
dgraph-alpha:
|
||||
image: dgraph/dgraph:v23.1.0
|
||||
container_name: orly-benchmark-dgraph-alpha
|
||||
working_dir: /data/alpha
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "9080:9080"
|
||||
command: dgraph alpha --my=dgraph-alpha:7080 --zero=dgraph-zero:5080 --security whitelist=0.0.0.0/0
|
||||
networks:
|
||||
- orly-benchmark
|
||||
depends_on:
|
||||
dgraph-zero:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "sh", "-c", "dgraph version || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 6
|
||||
start_period: 10s
|
||||
|
||||
networks:
|
||||
orly-benchmark:
|
||||
name: orly-benchmark-network
|
||||
driver: bridge
|
||||
Reference in New Issue
Block a user