diff --git a/main.go b/main.go index e422295..6bc63d5 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,8 @@ import ( "os/exec" "os/signal" "runtime" + "sync" + "syscall" "time" "github.com/pkg/profile" @@ -21,6 +23,7 @@ import ( "next.orly.dev/pkg/database" "next.orly.dev/pkg/encoders/hex" "next.orly.dev/pkg/spider" + "next.orly.dev/pkg/utils/interrupt" "next.orly.dev/pkg/version" ) @@ -83,16 +86,31 @@ func main() { log.I.F("enabling HTTP pprof server to support web viewer") cfg.PprofHTTP = true } + // Ensure profiling is stopped on interrupts (SIGINT/SIGTERM) as well as on normal exit + var profileStopOnce sync.Once + profileStop := func() {} switch cfg.Pprof { case "cpu": if cfg.PprofPath != "" { prof := profile.Start( profile.CPUProfile, profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("cpu profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.CPUProfile) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("cpu profiling stopped and flushed") + }) + } + defer profileStop() } case "memory": if cfg.PprofPath != "" { @@ -100,10 +118,22 @@ func main() { profile.MemProfile, profile.MemProfileRate(32), profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("memory profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.MemProfile) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("memory profiling stopped and flushed") + }) + } + defer profileStop() } case "allocation": if cfg.PprofPath != "" { @@ -111,30 +141,66 @@ func main() { profile.MemProfileAllocs, profile.MemProfileRate(32), profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("allocation profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.MemProfileAllocs) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("allocation profiling stopped and flushed") + }) + } + defer profileStop() } case "heap": if cfg.PprofPath != "" { prof := profile.Start( profile.MemProfileHeap, profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("heap profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.MemProfileHeap) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("heap profiling stopped and flushed") + }) + } + defer profileStop() } case "mutex": if cfg.PprofPath != "" { prof := profile.Start( profile.MutexProfile, profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("mutex profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.MutexProfile) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("mutex profiling stopped and flushed") + }) + } + defer profileStop() } case "threadcreate": if cfg.PprofPath != "" { @@ -142,33 +208,75 @@ func main() { profile.ThreadcreationProfile, profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("threadcreate profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.ThreadcreationProfile) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("threadcreate profiling stopped and flushed") + }) + } + defer profileStop() } case "goroutine": if cfg.PprofPath != "" { prof := profile.Start( profile.GoroutineProfile, profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("goroutine profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.GoroutineProfile) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("goroutine profiling stopped and flushed") + }) + } + defer profileStop() } case "block": if cfg.PprofPath != "" { prof := profile.Start( profile.BlockProfile, profile.ProfilePath(cfg.PprofPath), ) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("block profiling stopped and flushed") + }) + } + defer profileStop() } else { prof := profile.Start(profile.BlockProfile) - defer prof.Stop() + profileStop = func() { + profileStopOnce.Do(func() { + prof.Stop() + log.I.F("block profiling stopped and flushed") + }) + } + defer profileStop() } } + + // Register a handler so profiling is stopped when an interrupt is received + interrupt.AddHandler(func() { + log.I.F("interrupt received: stopping profiling") + profileStop() + }) ctx, cancel := context.WithCancel(context.Background()) var db *database.D if db, err = database.New( @@ -277,7 +385,7 @@ func main() { quit := app.Run(ctx, cfg, db) sigs := make(chan os.Signal, 1) - signal.Notify(sigs, os.Interrupt) + signal.Notify(sigs, os.Interrupt, syscall.SIGTERM) for { select { case <-sigs: @@ -296,5 +404,5 @@ func main() { return } } - log.I.F("exiting") + // log.I.F("exiting") } diff --git a/scripts/run-relay-pprof.sh b/scripts/run-relay-pprof.sh new file mode 100755 index 0000000..3d6f490 --- /dev/null +++ b/scripts/run-relay-pprof.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the relay with CPU profiling enabled, wait 60s, then open the +# generated profile using `go tool pprof` web UI. +# +# Notes: +# - Builds a temporary relay binary in /tmp and deletes it on exit. +# - Uses the exact env requested, plus ORLY_PPROF=cpu and a deterministic +# ORLY_PPROF_PATH inside a temp dir. +# - Profiles for DURATION seconds (default 60). +# - Launches pprof web UI at http://localhost:8000 and attempts to open browser. + +DURATION="${DURATION:-60}" +HEALTH_PORT="${HEALTH_PORT:-18081}" +ROOT_DIR="/home/mleku/src/next.orly.dev" +LISTEN_HOST="${LISTEN_HOST:-10.0.0.2}" + +cd "$ROOT_DIR" + +# Refresh embedded web assets +reset || true +./scripts/update-embedded-web.sh || true + +TMP_DIR="$(mktemp -d -t orly-pprof-XXXXXX)" +BIN_PATH="$TMP_DIR/next.orly.dev" +LOG_FILE="$TMP_DIR/relay.log" +PPROF_FILE="" +RELAY_PID="" +PPROF_DIR="$TMP_DIR/profiles" +mkdir -p "$PPROF_DIR" + +cleanup() { + # Try to stop relay if still running + if [[ -n "${RELAY_PID}" ]] && kill -0 "${RELAY_PID}" 2>/dev/null; then + kill "${RELAY_PID}" || true + wait "${RELAY_PID}" || true + fi + rm -f "$BIN_PATH" 2>/dev/null || true + rm -rf "$TMP_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +echo "[run-relay-pprof] Building relay binary ..." +GOFLAGS="${GOFLAGS:-}" go build -o "$BIN_PATH" . + +echo "[run-relay-pprof] Starting relay with CPU profiling ..." +( + ORLY_LOG_LEVEL=debug \ + ORLY_LISTEN="$LISTEN_HOST" \ + ORLY_PORT=3334 \ + ORLY_ADMINS=npub1fjqqy4a93z5zsjwsfxqhc2764kvykfdyttvldkkkdera8dr78vhsmmleku \ + ORLY_ACL_MODE=follows \ + ORLY_SPIDER_MODE=none \ + ORLY_RELAY_ADDRESSES=test.orly.dev \ + ORLY_IP_BLACKLIST=192.71.213.188 \ + ORLY_HEALTH_PORT="$HEALTH_PORT" \ + ORLY_ENABLE_SHUTDOWN=true \ + ORLY_PPROF_HTTP=true \ + ORLY_OPEN_PPROF_WEB=true \ + "$BIN_PATH" +) >"$LOG_FILE" 2>&1 & +RELAY_PID=$! + +# Wait for pprof HTTP server readiness +PPROF_BASE="http://${LISTEN_HOST}:6060" +echo "[run-relay-pprof] Waiting for pprof at ${PPROF_BASE} ..." +for i in {1..100}; do + if curl -fsS "${PPROF_BASE}/debug/pprof/" -o /dev/null 2>/dev/null; then + READY=1 + break + fi + sleep 0.2 +done +if [[ -z "${READY:-}" ]]; then + echo "[run-relay-pprof] ERROR: pprof HTTP server not reachable at ${PPROF_BASE}." >&2 + echo "[run-relay-pprof] Check that ${LISTEN_HOST} is a local bindable address." >&2 + # Attempt to dump recent logs for context + tail -n 100 "$LOG_FILE" || true + # Try INT to clean up + killall -INT next.orly.dev 2>/dev/null || true + exit 1 +fi + +# Open the HTTP pprof UI +( xdg-open "${PPROF_BASE}/debug/pprof/" >/dev/null 2>&1 || true ) & + +echo "[run-relay-pprof] Collecting CPU profile via HTTP for ${DURATION}s ..." +# The HTTP /debug/pprof/profile endpoint records CPU for the provided seconds +# and returns a pprof file without needing to stop the process. +curl -fsS --max-time $((DURATION+10)) \ + "${PPROF_BASE}/debug/pprof/profile?seconds=${DURATION}" \ + -o "$PPROF_DIR/cpu.pprof" || true + +echo "[run-relay-pprof] Sending SIGINT (Ctrl+C) for graceful shutdown ..." +killall -INT next.orly.dev 2>/dev/null || true + +# Wait up to ~60s for graceful shutdown so defers (pprof Stop) can run +for i in {1..300}; do + if ! pgrep -x next.orly.dev >/dev/null 2>&1; then + break + fi + sleep 0.2 +done + +# Try HTTP shutdown if still running (ensures defer paths can run) +if pgrep -x next.orly.dev >/dev/null 2>&1; then + echo "[run-relay-pprof] Still running, requesting /shutdown ..." + curl -fsS --max-time 2 "http://10.0.0.2:${HEALTH_PORT}/shutdown" >/dev/null 2>&1 || true + for i in {1..150}; do + if ! pgrep -x next.orly.dev >/dev/null 2>&1; then + break + fi + sleep 0.2 + done +fi +if pgrep -x next.orly.dev >/dev/null 2>&1; then + echo "[run-relay-pprof] Escalating: sending SIGTERM ..." + killall -TERM next.orly.dev 2>/dev/null || true + for i in {1..150}; do + if ! pgrep -x next.orly.dev >/dev/null 2>&1; then + break + fi + sleep 0.2 + done +fi +if pgrep -x next.orly.dev >/dev/null 2>&1; then + echo "[run-relay-pprof] Force kill: sending SIGKILL ..." + killall -KILL next.orly.dev 2>/dev/null || true +fi + +PPROF_FILE="$PPROF_DIR/cpu.pprof" +if [[ ! -s "$PPROF_FILE" ]]; then + echo "[run-relay-pprof] ERROR: HTTP CPU profile not captured (file empty)." >&2 + echo "[run-relay-pprof] Hint: Ensure ORLY_PPROF_HTTP=true and port 6060 is reachable." >&2 + exit 1 +fi + +echo "[run-relay-pprof] Detected profile file: $PPROF_FILE" +echo "[run-relay-pprof] Launching 'go tool pprof' web UI on :8000 ..." + +# Try to open a browser automatically, ignore failures +( sleep 0.6; xdg-open "http://localhost:8000" >/dev/null 2>&1 || true ) & + +exec go tool pprof -http=:8000 "$BIN_PATH" "$PPROF_FILE" + +