diff --git a/cmd/benchmark/Dockerfile b/cmd/benchmark/Dockerfile index 4ad7cee..91f48a8 100644 --- a/cmd/benchmark/Dockerfile +++ b/cmd/benchmark/Dockerfile @@ -30,23 +30,21 @@ RUN GOTOOLCHAIN=auto CGO_ENABLED=1 go build -tags minimal_log -o orly-relay . WORKDIR /relays -RUN git clone https://github.com/fiatjaf/khatru.git && \ - cd khatru/examples/basic-sqlite3 && \ - GOTOOLCHAIN=auto go build -o /relays/khatru-relay - -RUN git clone https://github.com/fiatjaf/relayer.git && \ - cd relayer && \ - git checkout 125f2a8 && \ - cd examples/basic && \ - GOTOOLCHAIN=auto go build -o /relays/relayer-bin - -RUN apt-get update && apt-get install -y libflatbuffers-dev libzstd-dev && \ +RUN apt-get update && apt-get install -y libflatbuffers-dev libzstd-dev zlib1g-dev && \ git clone https://github.com/hoytech/strfry.git && \ cd strfry && \ git submodule update --init && \ make setup-golpe && \ - make -j$(nproc) && \ - cp strfry /relays/strfry + make -j$(nproc) + +RUN git clone https://github.com/fiatjaf/khatru.git && \ + cd khatru/examples/basic-sqlite3 && \ + GOTOOLCHAIN=auto go build -o /relays/khatru-relay + +RUN git clone https://github.com/mleku/relayer.git && \ + cd relayer && \ + cd examples/basic && \ + GOTOOLCHAIN=auto go build -o /relays/relayer-bin FROM debian:bookworm-slim diff --git a/cmd/benchmark/README.md b/cmd/benchmark/README.md index 378f2b1..efd2782 100644 --- a/cmd/benchmark/README.md +++ b/cmd/benchmark/README.md @@ -4,13 +4,13 @@ A comprehensive performance benchmarking suite for Nostr relay implementations, ## Features -- **Multi-relay comparison benchmarks** - Compare Khatru, Strfry, Relayer, and Orly -- **Publishing performance testing** - Measure event ingestion rates and bandwidth -- **Query profiling** - Test various filter patterns and query speeds -- **Load pattern simulation** - Constant, spike, burst, sine, and ramp patterns -- **Timing instrumentation** - Track full event lifecycle and identify bottlenecks -- **Concurrent stress testing** - Multiple publishers with connection pooling -- **Production-grade event generation** - Proper secp256k1 signatures and UTF-8 content +- **Multi-relay comparison benchmarks** - Compare Khatru, Strfry, Relayer, and Orly +- **Publishing performance testing** - Measure event ingestion rates and bandwidth +- **Query profiling** - Test various filter patterns and query speeds +- **Load pattern simulation** - Constant, spike, burst, sine, and ramp patterns +- **Timing instrumentation** - Track full event lifecycle and identify bottlenecks +- **Concurrent stress testing** - Multiple publishers with connection pooling +- **Production-grade event generation** - Proper secp256k1 signatures and UTF-8 content - **Comparative reporting** - Markdown, JSON, and CSV format reports ## Prerequisites @@ -62,7 +62,7 @@ See [RELAY_COMPARISON_RESULTS.md](RELAY_COMPARISON_RESULTS.md) for detailed anal The docker-compose setup includes: - `orly`: Orly relay on port 7447 -- `khatru`: Khatru relay on port 7448 +- `khatru`: Khatru relay on port 7448 - `strfry`: Strfry relay on port 7450 - `relayer`: Relayer on port 7449 (with PostgreSQL) - `postgres`: PostgreSQL database for Relayer @@ -232,4 +232,4 @@ The benchmark suite consists of several components: - Events are generated with valid UTF-8 content to ensure compatibility - Connection pooling is used for realistic concurrent load testing - Query patterns test real-world filter combinations -- Docker setup includes all necessary dependencies and configurations \ No newline at end of file +- Docker setup includes all necessary dependencies and configurations diff --git a/cmd/benchmark/RELAY_COMPARISON_RESULTS.md b/cmd/benchmark/RELAY_COMPARISON_RESULTS.md index c66a7b8..a83e127 100644 --- a/cmd/benchmark/RELAY_COMPARISON_RESULTS.md +++ b/cmd/benchmark/RELAY_COMPARISON_RESULTS.md @@ -1,6 +1,6 @@ # Nostr Relay Performance Comparison -Benchmark results for Orly, Khatru, Strfry, and Relayer relay implementations. +Benchmark results for Khatru, Strfry, Relayer, and Orly relay implementations. ## Test Configuration @@ -36,14 +36,6 @@ Benchmark results for Orly, Khatru, Strfry, and Relayer relay implementations. ## Implementation Details -### Orly -- Language: Go -- Backend: Badger (embedded) -- Dependencies: Go 1.24+, libsecp256k1 -- Publishing: 7,731 events/sec, 1.29s duration -- Querying: 28 queries/sec, 3.57s duration -- **Note**: Performance drastically improved with logging disabled - ### Khatru - Language: Go - Backend: SQLite (embedded) diff --git a/cmd/benchmark/installer.go b/cmd/benchmark/installer.go new file mode 100644 index 0000000..2c48957 --- /dev/null +++ b/cmd/benchmark/installer.go @@ -0,0 +1,549 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" +) + +type DependencyType int + +const ( + Go DependencyType = iota + Rust + Cpp + Git + Make + Cmake + Pkg +) + +type RelayInstaller struct { + workDir string + installDir string + deps map[DependencyType]bool + mu sync.RWMutex + skipVerify bool +} + +func NewRelayInstaller(workDir, installDir string) *RelayInstaller { + return &RelayInstaller{ + workDir: workDir, + installDir: installDir, + deps: make(map[DependencyType]bool), + } +} + +func (ri *RelayInstaller) DetectDependencies() error { + deps := []struct { + dep DependencyType + cmd string + }{ + {Go, "go"}, + {Rust, "rustc"}, + {Cpp, "g++"}, + {Git, "git"}, + {Make, "make"}, + {Cmake, "cmake"}, + {Pkg, "pkg-config"}, + } + + ri.mu.Lock() + defer ri.mu.Unlock() + + for _, d := range deps { + _, err := exec.LookPath(d.cmd) + ri.deps[d.dep] = err == nil + } + + return nil +} + +func (ri *RelayInstaller) InstallMissingDependencies() error { + ri.mu.RLock() + missing := make([]DependencyType, 0) + for dep, exists := range ri.deps { + if !exists { + missing = append(missing, dep) + } + } + ri.mu.RUnlock() + + if len(missing) == 0 { + return nil + } + + switch runtime.GOOS { + case "linux": + return ri.installLinuxDeps(missing) + case "darwin": + return ri.installMacDeps(missing) + default: + return fmt.Errorf("unsupported OS: %s", runtime.GOOS) + } +} + +func (ri *RelayInstaller) installLinuxDeps(deps []DependencyType) error { + hasApt := ri.commandExists("apt-get") + hasYum := ri.commandExists("yum") + hasPacman := ri.commandExists("pacman") + + if !hasApt && !hasYum && !hasPacman { + return fmt.Errorf("no supported package manager found") + } + + if hasApt { + if err := ri.runCommand("sudo", "apt-get", "update"); err != nil { + return err + } + } + + for _, dep := range deps { + switch dep { + case Go: + if err := ri.installGo(); err != nil { + return err + } + case Rust: + if err := ri.installRust(); err != nil { + return err + } + default: + if hasApt { + if err := ri.installAptPackage(dep); err != nil { + return err + } + } else if hasYum { + if err := ri.installYumPackage(dep); err != nil { + return err + } + } else if hasPacman { + if err := ri.installPacmanPackage(dep); err != nil { + return err + } + } + } + } + + if err := ri.installSecp256k1(); err != nil { + return err + } + + return nil +} + +func (ri *RelayInstaller) installMacDeps(deps []DependencyType) error { + if !ri.commandExists("brew") { + return fmt.Errorf("homebrew not found, install from https://brew.sh") + } + + for _, dep := range deps { + switch dep { + case Go: + if err := ri.runCommand("brew", "install", "go"); err != nil { + return err + } + case Rust: + if err := ri.installRust(); err != nil { + return err + } + case Cpp: + if err := ri.runCommand("brew", "install", "gcc"); err != nil { + return err + } + case Git: + if err := ri.runCommand("brew", "install", "git"); err != nil { + return err + } + case Make: + if err := ri.runCommand("brew", "install", "make"); err != nil { + return err + } + case Cmake: + if err := ri.runCommand("brew", "install", "cmake"); err != nil { + return err + } + case Pkg: + if err := ri.runCommand("brew", "install", "pkg-config"); err != nil { + return err + } + } + } + + if err := ri.installSecp256k1(); err != nil { + return err + } + + return nil +} + +func (ri *RelayInstaller) installAptPackage(dep DependencyType) error { + var pkgName string + switch dep { + case Cpp: + pkgName = "build-essential" + case Git: + pkgName = "git" + case Make: + pkgName = "make" + case Cmake: + pkgName = "cmake" + case Pkg: + pkgName = "pkg-config" + default: + return nil + } + + return ri.runCommand("sudo", "apt-get", "install", "-y", pkgName, "autotools-dev", "autoconf", "libtool") +} + +func (ri *RelayInstaller) installYumPackage(dep DependencyType) error { + var pkgName string + switch dep { + case Cpp: + pkgName = "gcc-c++" + case Git: + pkgName = "git" + case Make: + pkgName = "make" + case Cmake: + pkgName = "cmake" + case Pkg: + pkgName = "pkgconfig" + default: + return nil + } + + return ri.runCommand("sudo", "yum", "install", "-y", pkgName) +} + +func (ri *RelayInstaller) installPacmanPackage(dep DependencyType) error { + var pkgName string + switch dep { + case Cpp: + pkgName = "gcc" + case Git: + pkgName = "git" + case Make: + pkgName = "make" + case Cmake: + pkgName = "cmake" + case Pkg: + pkgName = "pkgconf" + default: + return nil + } + + return ri.runCommand("sudo", "pacman", "-S", "--noconfirm", pkgName) +} + +func (ri *RelayInstaller) installGo() error { + version := "1.21.5" + arch := runtime.GOARCH + if arch == "amd64" { + arch = "amd64" + } else if arch == "arm64" { + arch = "arm64" + } + + filename := fmt.Sprintf("go%s.%s-%s.tar.gz", version, runtime.GOOS, arch) + url := fmt.Sprintf("https://golang.org/dl/%s", filename) + + tmpFile := filepath.Join(os.TempDir(), filename) + if err := ri.runCommand("wget", "-O", tmpFile, url); err != nil { + return fmt.Errorf("failed to download Go: %w", err) + } + + if err := ri.runCommand("sudo", "tar", "-C", "/usr/local", "-xzf", tmpFile); err != nil { + return fmt.Errorf("failed to extract Go: %w", err) + } + + os.Remove(tmpFile) + + profile := filepath.Join(os.Getenv("HOME"), ".profile") + f, err := os.OpenFile(profile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err == nil { + f.WriteString("\nexport PATH=$PATH:/usr/local/go/bin\n") + f.Close() + } + + return nil +} + +func (ri *RelayInstaller) installRust() error { + return ri.runCommand("curl", "--proto", "=https", "--tlsv1.2", "-sSf", "https://sh.rustup.rs", "|", "sh", "-s", "--", "-y") +} + +func (ri *RelayInstaller) installSecp256k1() error { + switch runtime.GOOS { + case "linux": + if ri.commandExists("apt-get") { + if err := ri.runCommand("sudo", "apt-get", "install", "-y", "libsecp256k1-dev"); err != nil { + return ri.buildSecp256k1FromSource() + } + return nil + } else if ri.commandExists("yum") { + if err := ri.runCommand("sudo", "yum", "install", "-y", "libsecp256k1-devel"); err != nil { + return ri.buildSecp256k1FromSource() + } + return nil + } else if ri.commandExists("pacman") { + if err := ri.runCommand("sudo", "pacman", "-S", "--noconfirm", "libsecp256k1"); err != nil { + return ri.buildSecp256k1FromSource() + } + return nil + } + return ri.buildSecp256k1FromSource() + case "darwin": + if err := ri.runCommand("brew", "install", "libsecp256k1"); err != nil { + return ri.buildSecp256k1FromSource() + } + return nil + default: + return ri.buildSecp256k1FromSource() + } +} + +func (ri *RelayInstaller) buildSecp256k1FromSource() error { + secp256k1Dir := filepath.Join(ri.workDir, "secp256k1") + + if err := ri.runCommand("git", "clone", "https://github.com/bitcoin-core/secp256k1.git", secp256k1Dir); err != nil { + return fmt.Errorf("failed to clone secp256k1: %w", err) + } + + if err := os.Chdir(secp256k1Dir); err != nil { + return err + } + + if err := ri.runCommand("./autogen.sh"); err != nil { + return fmt.Errorf("failed to run autogen: %w", err) + } + + configArgs := []string{"--enable-module-schnorrsig", "--enable-module-recovery"} + if err := ri.runCommand("./configure", configArgs...); err != nil { + return fmt.Errorf("failed to configure secp256k1: %w", err) + } + + if err := ri.runCommand("make"); err != nil { + return fmt.Errorf("failed to build secp256k1: %w", err) + } + + if err := ri.runCommand("sudo", "make", "install"); err != nil { + return fmt.Errorf("failed to install secp256k1: %w", err) + } + + if err := ri.runCommand("sudo", "ldconfig"); err != nil && runtime.GOOS == "linux" { + return fmt.Errorf("failed to run ldconfig: %w", err) + } + + return nil +} + +func (ri *RelayInstaller) InstallKhatru() error { + khatruDir := filepath.Join(ri.workDir, "khatru") + + if err := ri.runCommand("git", "clone", "https://github.com/fiatjaf/khatru.git", khatruDir); err != nil { + return fmt.Errorf("failed to clone khatru: %w", err) + } + + if err := os.Chdir(khatruDir); err != nil { + return err + } + + if err := ri.runCommand("go", "mod", "tidy"); err != nil { + return fmt.Errorf("failed to tidy khatru: %w", err) + } + + binPath := filepath.Join(ri.installDir, "khatru") + if err := ri.runCommand("go", "build", "-o", binPath, "."); err != nil { + return fmt.Errorf("failed to build khatru: %w", err) + } + + return nil +} + +func (ri *RelayInstaller) InstallRelayer() error { + relayerDir := filepath.Join(ri.workDir, "relayer") + + if err := ri.runCommand("git", "clone", "https://github.com/fiatjaf/relayer.git", relayerDir); err != nil { + return fmt.Errorf("failed to clone relayer: %w", err) + } + + if err := os.Chdir(relayerDir); err != nil { + return err + } + + if err := ri.runCommand("go", "mod", "tidy"); err != nil { + return fmt.Errorf("failed to tidy relayer: %w", err) + } + + binPath := filepath.Join(ri.installDir, "relayer") + if err := ri.runCommand("go", "build", "-o", binPath, "."); err != nil { + return fmt.Errorf("failed to build relayer: %w", err) + } + + return nil +} + +func (ri *RelayInstaller) InstallStrfry() error { + strfryDir := filepath.Join(ri.workDir, "strfry") + + if err := ri.runCommand("git", "clone", "https://github.com/hoytech/strfry.git", strfryDir); err != nil { + return fmt.Errorf("failed to clone strfry: %w", err) + } + + if err := os.Chdir(strfryDir); err != nil { + return err + } + + if err := ri.runCommand("git", "submodule", "update", "--init"); err != nil { + return fmt.Errorf("failed to init submodules: %w", err) + } + + if err := ri.runCommand("make", "setup-golpe"); err != nil { + return fmt.Errorf("failed to setup golpe: %w", err) + } + + if err := ri.runCommand("make"); err != nil { + return fmt.Errorf("failed to build strfry: %w", err) + } + + srcBin := filepath.Join(strfryDir, "strfry") + dstBin := filepath.Join(ri.installDir, "strfry") + if err := ri.runCommand("cp", srcBin, dstBin); err != nil { + return fmt.Errorf("failed to copy strfry binary: %w", err) + } + + return nil +} + +func (ri *RelayInstaller) InstallRustRelay() error { + rustRelayDir := filepath.Join(ri.workDir, "nostr-rs-relay") + + if err := ri.runCommand("git", "clone", "https://github.com/scsibug/nostr-rs-relay.git", rustRelayDir); err != nil { + return fmt.Errorf("failed to clone rust relay: %w", err) + } + + if err := os.Chdir(rustRelayDir); err != nil { + return err + } + + if err := ri.runCommand("cargo", "build", "--release"); err != nil { + return fmt.Errorf("failed to build rust relay: %w", err) + } + + srcBin := filepath.Join(rustRelayDir, "target", "release", "nostr-rs-relay") + dstBin := filepath.Join(ri.installDir, "nostr-rs-relay") + if err := ri.runCommand("cp", srcBin, dstBin); err != nil { + return fmt.Errorf("failed to copy rust relay binary: %w", err) + } + + return nil +} + +func (ri *RelayInstaller) VerifyInstallation() error { + if ri.skipVerify { + return nil + } + + binaries := []string{"khatru", "relayer", "strfry", "nostr-rs-relay"} + + for _, binary := range binaries { + binPath := filepath.Join(ri.installDir, binary) + if _, err := os.Stat(binPath); os.IsNotExist(err) { + return fmt.Errorf("binary %s not found at %s", binary, binPath) + } + + if err := ri.runCommand("chmod", "+x", binPath); err != nil { + return fmt.Errorf("failed to make %s executable: %w", binary, err) + } + } + + return nil +} + +func (ri *RelayInstaller) commandExists(cmd string) bool { + _, err := exec.LookPath(cmd) + return err == nil +} + +func (ri *RelayInstaller) runCommand(name string, args ...string) error { + if name == "curl" && len(args) > 0 && strings.Contains(strings.Join(args, " "), "|") { + fullCmd := fmt.Sprintf("%s %s", name, strings.Join(args, " ")) + cmd := exec.Command("bash", "-c", fullCmd) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + + cmd := exec.Command(name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func (ri *RelayInstaller) InstallSecp256k1Only() error { + fmt.Println("Installing secp256k1 library...") + + if err := os.MkdirAll(ri.workDir, 0755); err != nil { + return err + } + + if err := ri.installSecp256k1(); err != nil { + return fmt.Errorf("failed to install secp256k1: %w", err) + } + + fmt.Println("secp256k1 installed successfully") + return nil +} + +func (ri *RelayInstaller) InstallAll() error { + fmt.Println("Detecting dependencies...") + if err := ri.DetectDependencies(); err != nil { + return err + } + + fmt.Println("Installing missing dependencies...") + if err := ri.InstallMissingDependencies(); err != nil { + return err + } + + if err := os.MkdirAll(ri.workDir, 0755); err != nil { + return err + } + if err := os.MkdirAll(ri.installDir, 0755); err != nil { + return err + } + + fmt.Println("Installing khatru...") + if err := ri.InstallKhatru(); err != nil { + return err + } + + fmt.Println("Installing relayer...") + if err := ri.InstallRelayer(); err != nil { + return err + } + + fmt.Println("Installing strfry...") + if err := ri.InstallStrfry(); err != nil { + return err + } + + fmt.Println("Installing rust relay...") + if err := ri.InstallRustRelay(); err != nil { + return err + } + + fmt.Println("Verifying installation...") + if err := ri.VerifyInstallation(); err != nil { + return err + } + + fmt.Println("All relays installed successfully") + return nil +} diff --git a/cmd/benchmark/run.sh b/cmd/benchmark/run.sh new file mode 100755 index 0000000..98fd1dd --- /dev/null +++ b/cmd/benchmark/run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +docker compose up -d --remove-orphans +docker compose run benchmark -relay ws://orly:7447 -events 10000 -queries 100 +docker compose run benchmark -relay ws://khatru:7447 -events 10000 -queries 100 +docker compose run benchmark -relay ws://strfry:7777 -events 10000 -queries 100 +docker compose run benchmark -relay ws://relayer:7447 -events 10000 -queries 100 diff --git a/cmd/benchmark/simple_event.go b/cmd/benchmark/simple_event.go index 7f0a0ac..5961c67 100644 --- a/cmd/benchmark/simple_event.go +++ b/cmd/benchmark/simple_event.go @@ -13,7 +13,7 @@ import ( func generateSimpleEvent(signer *testSigner, contentSize int) *event.E { content := generateContent(contentSize) - + ev := &event.E{ Kind: kind.TextNote, Tags: tags.New(), @@ -21,11 +21,11 @@ func generateSimpleEvent(signer *testSigner, contentSize int) *event.E { CreatedAt: timestamp.Now(), Pubkey: signer.Pub(), } - + if err := ev.Sign(signer); chk.E(err) { panic(fmt.Sprintf("failed to sign event: %v", err)) } - + return ev } @@ -42,7 +42,7 @@ func generateContent(size int) string { "back", "after", "use", "two", "how", "our", "work", "first", "well", "way", "even", "new", "want", "because", "any", "these", "give", "day", "most", "us", } - + result := "" for len(result) < size { if len(result) > 0 { @@ -50,10 +50,10 @@ func generateContent(size int) string { } result += words[frand.Intn(len(words))] } - + if len(result) > size { result = result[:size] } - + return result -} +} \ No newline at end of file diff --git a/cmd/benchmark/test_signer.go b/cmd/benchmark/test_signer.go index 8d6544c..c366c97 100644 --- a/cmd/benchmark/test_signer.go +++ b/cmd/benchmark/test_signer.go @@ -18,4 +18,4 @@ func newTestSigner() *testSigner { return &testSigner{Signer: s} } -var _ signer.I = (*testSigner)(nil) +var _ signer.I = (*testSigner)(nil) \ No newline at end of file