Migrate internal module imports to unified package path.
Replaced legacy `*.orly` module imports with `next.orly.dev/pkg` paths across the codebase for consistency. Removed legacy `go.mod` files from sub-packages, consolidating dependency management. Added Dockerfiles and configurations for benchmarking environments.
This commit is contained in:
265
cmd/benchmark/benchmark-runner.sh
Normal file
265
cmd/benchmark/benchmark-runner.sh
Normal file
@@ -0,0 +1,265 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Benchmark runner script for testing multiple Nostr relay implementations
|
||||
# This script coordinates testing all relays and aggregates results
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration from environment variables
|
||||
BENCHMARK_EVENTS="${BENCHMARK_EVENTS:-10000}"
|
||||
BENCHMARK_WORKERS="${BENCHMARK_WORKERS:-8}"
|
||||
BENCHMARK_DURATION="${BENCHMARK_DURATION:-60s}"
|
||||
BENCHMARK_TARGETS="${BENCHMARK_TARGETS:-next-orly:8001,khatru-sqlite:8002,khatru-badger:8003,relayer-basic:8004,strfry:8005,nostr-rs-relay:8006}"
|
||||
OUTPUT_DIR="${OUTPUT_DIR:-/reports}"
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "${OUTPUT_DIR}"
|
||||
|
||||
# Generate timestamp for this benchmark run
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
RUN_DIR="${OUTPUT_DIR}/run_${TIMESTAMP}"
|
||||
mkdir -p "${RUN_DIR}"
|
||||
|
||||
echo "=================================================="
|
||||
echo "Nostr Relay Benchmark Suite"
|
||||
echo "=================================================="
|
||||
echo "Timestamp: $(date)"
|
||||
echo "Events per test: ${BENCHMARK_EVENTS}"
|
||||
echo "Concurrent workers: ${BENCHMARK_WORKERS}"
|
||||
echo "Test duration: ${BENCHMARK_DURATION}"
|
||||
echo "Output directory: ${RUN_DIR}"
|
||||
echo "=================================================="
|
||||
|
||||
# Function to wait for relay to be ready
|
||||
wait_for_relay() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local max_attempts=60
|
||||
local attempt=0
|
||||
|
||||
echo "Waiting for ${name} to be ready at ${url}..."
|
||||
|
||||
while [ $attempt -lt $max_attempts ]; do
|
||||
if wget --quiet --tries=1 --spider --timeout=5 "http://${url}" 2>/dev/null || \
|
||||
curl -f --connect-timeout 5 --max-time 5 "http://${url}" >/dev/null 2>&1; then
|
||||
echo "${name} is ready!"
|
||||
return 0
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
echo " Attempt ${attempt}/${max_attempts}: ${name} not ready yet..."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "ERROR: ${name} failed to become ready after ${max_attempts} attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Function to run benchmark against a specific relay
|
||||
run_benchmark() {
|
||||
local relay_name="$1"
|
||||
local relay_url="$2"
|
||||
local output_file="$3"
|
||||
|
||||
echo ""
|
||||
echo "=================================================="
|
||||
echo "Testing ${relay_name} at ws://${relay_url}"
|
||||
echo "=================================================="
|
||||
|
||||
# Wait for relay to be ready
|
||||
if ! wait_for_relay "${relay_name}" "${relay_url}"; then
|
||||
echo "ERROR: ${relay_name} is not responding, skipping..."
|
||||
echo "RELAY: ${relay_name}" > "${output_file}"
|
||||
echo "STATUS: FAILED - Relay not responding" >> "${output_file}"
|
||||
echo "ERROR: Connection failed" >> "${output_file}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Run the benchmark
|
||||
echo "Running benchmark against ${relay_name}..."
|
||||
|
||||
# Create temporary directory for this relay's data
|
||||
TEMP_DATA_DIR="/tmp/benchmark_${relay_name}_$$"
|
||||
mkdir -p "${TEMP_DATA_DIR}"
|
||||
|
||||
# Run benchmark and capture both stdout and stderr
|
||||
if /app/benchmark \
|
||||
-datadir="${TEMP_DATA_DIR}" \
|
||||
-events="${BENCHMARK_EVENTS}" \
|
||||
-workers="${BENCHMARK_WORKERS}" \
|
||||
-duration="${BENCHMARK_DURATION}" \
|
||||
> "${output_file}" 2>&1; then
|
||||
|
||||
echo "✓ Benchmark completed successfully for ${relay_name}"
|
||||
|
||||
# Add relay identification to the report
|
||||
echo "" >> "${output_file}"
|
||||
echo "RELAY_NAME: ${relay_name}" >> "${output_file}"
|
||||
echo "RELAY_URL: ws://${relay_url}" >> "${output_file}"
|
||||
echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
|
||||
echo "BENCHMARK_CONFIG:" >> "${output_file}"
|
||||
echo " Events: ${BENCHMARK_EVENTS}" >> "${output_file}"
|
||||
echo " Workers: ${BENCHMARK_WORKERS}" >> "${output_file}"
|
||||
echo " Duration: ${BENCHMARK_DURATION}" >> "${output_file}"
|
||||
|
||||
else
|
||||
echo "✗ Benchmark failed for ${relay_name}"
|
||||
echo "" >> "${output_file}"
|
||||
echo "RELAY_NAME: ${relay_name}" >> "${output_file}"
|
||||
echo "RELAY_URL: ws://${relay_url}" >> "${output_file}"
|
||||
echo "STATUS: FAILED" >> "${output_file}"
|
||||
echo "TEST_TIMESTAMP: $(date -Iseconds)" >> "${output_file}"
|
||||
fi
|
||||
|
||||
# Clean up temporary data
|
||||
rm -rf "${TEMP_DATA_DIR}"
|
||||
}
|
||||
|
||||
# Function to generate aggregate report
|
||||
generate_aggregate_report() {
|
||||
local aggregate_file="${RUN_DIR}/aggregate_report.txt"
|
||||
|
||||
echo "Generating aggregate report..."
|
||||
|
||||
cat > "${aggregate_file}" << EOF
|
||||
================================================================
|
||||
NOSTR RELAY BENCHMARK AGGREGATE REPORT
|
||||
================================================================
|
||||
Generated: $(date -Iseconds)
|
||||
Benchmark Configuration:
|
||||
Events per test: ${BENCHMARK_EVENTS}
|
||||
Concurrent workers: ${BENCHMARK_WORKERS}
|
||||
Test duration: ${BENCHMARK_DURATION}
|
||||
|
||||
Relays tested: $(echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | wc -l)
|
||||
|
||||
================================================================
|
||||
SUMMARY BY RELAY
|
||||
================================================================
|
||||
|
||||
EOF
|
||||
|
||||
# Process each relay's results
|
||||
echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | while IFS=':' read -r relay_name relay_port; do
|
||||
if [ -z "${relay_name}" ] || [ -z "${relay_port}" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
relay_file="${RUN_DIR}/${relay_name}_results.txt"
|
||||
|
||||
echo "Relay: ${relay_name}" >> "${aggregate_file}"
|
||||
echo "----------------------------------------" >> "${aggregate_file}"
|
||||
|
||||
if [ -f "${relay_file}" ]; then
|
||||
# Extract key metrics from the relay's report
|
||||
if grep -q "STATUS: FAILED" "${relay_file}"; then
|
||||
echo "Status: FAILED" >> "${aggregate_file}"
|
||||
grep "ERROR:" "${relay_file}" | head -1 >> "${aggregate_file}" || echo "Error: Unknown failure" >> "${aggregate_file}"
|
||||
else
|
||||
echo "Status: COMPLETED" >> "${aggregate_file}"
|
||||
|
||||
# Extract performance metrics
|
||||
grep "Events/sec:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
|
||||
grep "Success Rate:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
|
||||
grep "Avg Latency:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
|
||||
grep "P95 Latency:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
|
||||
grep "Memory:" "${relay_file}" | head -3 >> "${aggregate_file}" || true
|
||||
fi
|
||||
else
|
||||
echo "Status: NO RESULTS FILE" >> "${aggregate_file}"
|
||||
echo "Error: Results file not found" >> "${aggregate_file}"
|
||||
fi
|
||||
|
||||
echo "" >> "${aggregate_file}"
|
||||
done
|
||||
|
||||
cat >> "${aggregate_file}" << EOF
|
||||
|
||||
================================================================
|
||||
DETAILED RESULTS
|
||||
================================================================
|
||||
|
||||
Individual relay reports are available in:
|
||||
$(ls "${RUN_DIR}"/*_results.txt 2>/dev/null | sed 's|^| - |' || echo " No individual reports found")
|
||||
|
||||
================================================================
|
||||
BENCHMARK COMPARISON TABLE
|
||||
================================================================
|
||||
|
||||
EOF
|
||||
|
||||
# Create a comparison table
|
||||
printf "%-20s %-10s %-15s %-15s %-15s\n" "Relay" "Status" "Peak Tput/s" "Avg Latency" "Success Rate" >> "${aggregate_file}"
|
||||
printf "%-20s %-10s %-15s %-15s %-15s\n" "----" "------" "-----------" "-----------" "------------" >> "${aggregate_file}"
|
||||
|
||||
echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | while IFS=':' read -r relay_name relay_port; do
|
||||
if [ -z "${relay_name}" ] || [ -z "${relay_port}" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
relay_file="${RUN_DIR}/${relay_name}_results.txt"
|
||||
|
||||
if [ -f "${relay_file}" ]; then
|
||||
if grep -q "STATUS: FAILED" "${relay_file}"; then
|
||||
printf "%-20s %-10s %-15s %-15s %-15s\n" "${relay_name}" "FAILED" "-" "-" "-" >> "${aggregate_file}"
|
||||
else
|
||||
# Extract metrics for the table
|
||||
peak_tput=$(grep "Events/sec:" "${relay_file}" | head -1 | awk '{print $2}' || echo "-")
|
||||
avg_latency=$(grep "Avg Latency:" "${relay_file}" | head -1 | awk '{print $3}' || echo "-")
|
||||
success_rate=$(grep "Success Rate:" "${relay_file}" | head -1 | awk '{print $3}' || echo "-")
|
||||
|
||||
printf "%-20s %-10s %-15s %-15s %-15s\n" "${relay_name}" "OK" "${peak_tput}" "${avg_latency}" "${success_rate}" >> "${aggregate_file}"
|
||||
fi
|
||||
else
|
||||
printf "%-20s %-10s %-15s %-15s %-15s\n" "${relay_name}" "NO DATA" "-" "-" "-" >> "${aggregate_file}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "" >> "${aggregate_file}"
|
||||
echo "================================================================" >> "${aggregate_file}"
|
||||
echo "End of Report" >> "${aggregate_file}"
|
||||
echo "================================================================" >> "${aggregate_file}"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
echo "Starting relay benchmark suite..."
|
||||
|
||||
# Parse targets and run benchmarks
|
||||
echo "${BENCHMARK_TARGETS}" | tr ',' '\n' | while IFS=':' read -r relay_name relay_port; do
|
||||
if [ -z "${relay_name}" ] || [ -z "${relay_port}" ]; then
|
||||
echo "WARNING: Skipping invalid target: ${relay_name}:${relay_port}"
|
||||
continue
|
||||
fi
|
||||
|
||||
relay_url="${relay_name}:${relay_port}"
|
||||
output_file="${RUN_DIR}/${relay_name}_results.txt"
|
||||
|
||||
run_benchmark "${relay_name}" "${relay_url}" "${output_file}"
|
||||
|
||||
# Small delay between tests
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Generate aggregate report
|
||||
generate_aggregate_report
|
||||
|
||||
echo ""
|
||||
echo "=================================================="
|
||||
echo "Benchmark Suite Completed!"
|
||||
echo "=================================================="
|
||||
echo "Results directory: ${RUN_DIR}"
|
||||
echo "Aggregate report: ${RUN_DIR}/aggregate_report.txt"
|
||||
echo ""
|
||||
|
||||
# Display summary
|
||||
if [ -f "${RUN_DIR}/aggregate_report.txt" ]; then
|
||||
echo "Quick Summary:"
|
||||
echo "=============="
|
||||
grep -A 10 "BENCHMARK COMPARISON TABLE" "${RUN_DIR}/aggregate_report.txt" | tail -n +4
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "All benchmark files:"
|
||||
ls -la "${RUN_DIR}/"
|
||||
echo ""
|
||||
echo "Benchmark suite finished at: $(date)"
|
||||
Reference in New Issue
Block a user