From 2e865c9616b3b6e56440792194d4b6e53e53a6c5 Mon Sep 17 00:00:00 2001 From: mleku Date: Tue, 25 Nov 2025 06:03:22 +0000 Subject: [PATCH] fix workflow to fetch libsecp256k1.so --- .claude/settings.local.json | 3 ++- .gitea/workflows/go.yml | 14 +++++++++---- CLAUDE.md | 34 +++++++++++++++++-------------- Dockerfile | 7 +++++-- docs/LIBSECP256K1_DEPLOYMENT.md | 10 ++++++++- docs/PUREGO_BUILD_SYSTEM.md | 7 +++++-- pkg/version/version | 2 +- scripts/benchmark.sh | 9 +++++---- scripts/build-all-platforms.sh | 36 ++++++++++++++++++++------------- scripts/deploy.sh | 11 ++++++---- scripts/runtests.sh | 13 ++++++++++-- scripts/test.sh | 15 ++++++++++++-- 12 files changed, 109 insertions(+), 52 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 13a648b..33d8016 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -130,7 +130,8 @@ "Bash(sudo journalctl:*)", "Bash(systemctl:*)", "Bash(systemctl show:*)", - "Bash(ssh relay1:*)" + "Bash(ssh relay1:*)", + "Bash(done)" ], "deny": [], "ask": [] diff --git a/.gitea/workflows/go.yml b/.gitea/workflows/go.yml index bfbd040..989994b 100644 --- a/.gitea/workflows/go.yml +++ b/.gitea/workflows/go.yml @@ -55,8 +55,12 @@ jobs: export PATH=/usr/local/go/bin:$PATH cd ${GITHUB_WORKSPACE} echo "Running tests..." - # Copy the libsecp256k1.so to root directory so tests can find it - cp pkg/crypto/p8k/libsecp256k1.so . + # Download libsecp256k1.so from nostr repository + echo "Downloading libsecp256k1.so from nostr repository..." + wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so + chmod +x libsecp256k1.so + # Set LD_LIBRARY_PATH so tests can find the library + export LD_LIBRARY_PATH=${GITHUB_WORKSPACE}:${LD_LIBRARY_PATH} CGO_ENABLED=0 go test -v $(go list ./... | grep -v '/cmd/benchmark/external/' | xargs -n1 sh -c 'ls $0/*_test.go 1>/dev/null 2>&1 && echo $0' | grep .) || true - name: Build Release Binaries (Pure Go + purego) @@ -71,8 +75,10 @@ jobs: # Create directory for binaries mkdir -p release-binaries - # Copy the pre-compiled libsecp256k1.so for Linux AMD64 - cp pkg/crypto/p8k/libsecp256k1.so release-binaries/libsecp256k1-linux-amd64.so + # Download the pre-compiled libsecp256k1.so for Linux AMD64 from nostr repository + echo "Downloading libsecp256k1.so from nostr repository..." + wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O release-binaries/libsecp256k1-linux-amd64.so + chmod +x release-binaries/libsecp256k1-linux-amd64.so # Build for Linux AMD64 (pure Go + purego dynamic loading) echo "Building Linux AMD64 (pure Go + purego dynamic loading)..." diff --git a/CLAUDE.md b/CLAUDE.md index 2c86c30..559bf54 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -59,8 +59,10 @@ cd app/web && bun run dev # Or manually with purego setup CGO_ENABLED=0 go test ./... -# Note: libsecp256k1.so must be available for crypto tests -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)/pkg/crypto/p8k" +# Note: libsecp256k1.so is automatically downloaded by test.sh if needed +# It can also be manually downloaded from the nostr repository: +# wget https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so +# export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)" ``` ### Run Specific Package Tests @@ -92,8 +94,8 @@ go run cmd/relay-tester/main.go -url ws://localhost:3334 -test "Basic Event" # Run Go benchmarks in specific package go test -bench=. -benchmem ./pkg/database -# Crypto benchmarks -cd pkg/crypto/p8k && make bench +# Note: Crypto benchmarks are now in the external nostr library at: +# https://git.mleku.dev/mleku/nostr # Run full relay benchmark suite cd cmd/benchmark @@ -203,15 +205,15 @@ export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size - `hex/` - SIMD-accelerated hex encoding using templexxx/xhex - `timestamp/`, `kind/`, `tag/` - Specialized field encoders -**`pkg/crypto/`** - Cryptographic operations -- `p8k/` - Pure Go secp256k1 using purego (no CGO) to dynamically load libsecp256k1.so - - `secp.go` - Dynamic library loading and function binding - - `schnorr.go` - Schnorr signature operations (NIP-01) - - `ecdh.go` - ECDH for encrypted DMs (NIP-04, NIP-44) - - `recovery.go` - Public key recovery from signatures - - `libsecp256k1.so` - Pre-compiled secp256k1 library -- `keys/` - Key derivation and conversion utilities -- `sha256/` - SIMD-accelerated SHA256 using minio/sha256-simd +**Cryptographic operations** (from `git.mleku.dev/mleku/nostr` library) +- Pure Go secp256k1 using purego (no CGO) to dynamically load libsecp256k1.so + - Schnorr signature operations (NIP-01) + - ECDH for encrypted DMs (NIP-04, NIP-44) + - Public key recovery from signatures + - `libsecp256k1.so` - Downloaded from nostr repository at runtime/build time +- Key derivation and conversion utilities +- SIMD-accelerated SHA256 using minio/sha256-simd +- SIMD-accelerated hex encoding using templexxx/xhex **`pkg/acl/`** - Access control systems - `acl.go` - ACL registry and interface @@ -255,9 +257,11 @@ export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size **Pure Go with Purego:** - All builds use `CGO_ENABLED=0` -- The p8k crypto library uses `github.com/ebitengine/purego` to dynamically load `libsecp256k1.so` at runtime +- The p8k crypto library (from `git.mleku.dev/mleku/nostr`) uses `github.com/ebitengine/purego` to dynamically load `libsecp256k1.so` at runtime - This avoids CGO complexity while maintaining C library performance -- `libsecp256k1.so` must be in `LD_LIBRARY_PATH` or same directory as binary +- `libsecp256k1.so` is automatically downloaded by build/test scripts from the nostr repository +- Manual download: `wget https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so` +- Library must be in `LD_LIBRARY_PATH` or same directory as binary for runtime loading **Database Backend Selection:** - Supports multiple backends via `ORLY_DB_TYPE` environment variable diff --git a/Dockerfile b/Dockerfile index c42c350..b42504b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o orly -ldflags="-w -s" . FROM alpine:latest # Install runtime dependencies -RUN apk add --no-cache ca-certificates curl +RUN apk add --no-cache ca-certificates curl wget # Create app user RUN addgroup -g 1000 orly && \ @@ -34,7 +34,10 @@ WORKDIR /app # Copy binary from builder COPY --from=builder /build/orly /app/orly -COPY --from=builder /build/pkg/crypto/p8k/libsecp256k1.so /app/libsecp256k1.so + +# Download libsecp256k1.so from nostr repository (optional for performance) +RUN wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so \ + -O /app/libsecp256k1.so || echo "Warning: libsecp256k1.so download failed (optional)" # Set library path ENV LD_LIBRARY_PATH=/app diff --git a/docs/LIBSECP256K1_DEPLOYMENT.md b/docs/LIBSECP256K1_DEPLOYMENT.md index 74dbdaf..93b5b73 100644 --- a/docs/LIBSECP256K1_DEPLOYMENT.md +++ b/docs/LIBSECP256K1_DEPLOYMENT.md @@ -1,6 +1,14 @@ # libsecp256k1 Deployment Guide -All build scripts have been updated to ensure libsecp256k1.so is placed next to the executable. +> **NOTE (Updated 2025):** This project now uses pure Go with purego (no CGO). The crypto library is part of the external `git.mleku.dev/mleku/nostr` dependency. The `libsecp256k1.so` file is automatically downloaded from the nostr repository during build/test. See [CLAUDE.md](../CLAUDE.md) for current build instructions. + +## Current Approach (Pure Go + Purego) + +All build scripts download `libsecp256k1.so` from `https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so` and place it next to the executable for optimal performance. + +## Legacy Information (For Reference) + +The information below describes the previous CGO-based approach and is kept for historical reference. ## Updated Scripts diff --git a/docs/PUREGO_BUILD_SYSTEM.md b/docs/PUREGO_BUILD_SYSTEM.md index ae0581a..c59a68a 100644 --- a/docs/PUREGO_BUILD_SYSTEM.md +++ b/docs/PUREGO_BUILD_SYSTEM.md @@ -31,7 +31,7 @@ ORLY relay uses **pure Go builds (`CGO_ENABLED=0`)** across all platforms. The p ### Purego Dynamic Loading -The p8k library (`pkg/crypto/p8k`) uses purego to: +The p8k library (from `git.mleku.dev/mleku/nostr`) uses purego to: 1. **At build time**: Compile pure Go code (`CGO_ENABLED=0`) 2. **At runtime**: Attempt to dynamically load `libsecp256k1` @@ -287,8 +287,11 @@ RUN go build -ldflags "-s -w" -o orly . # Runtime can optionally include library FROM alpine:latest +RUN apk add --no-cache wget ca-certificates COPY --from=builder /build/orly /app/orly -COPY --from=builder /build/pkg/crypto/p8k/libsecp256k1.so /app/ || true +# Download libsecp256k1.so from nostr repository (optional for performance) +RUN wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so \ + -O /app/libsecp256k1.so || echo "Warning: libsecp256k1.so download failed (optional)" ENV LD_LIBRARY_PATH=/app CMD ["/app/orly"] ``` diff --git a/pkg/version/version b/pkg/version/version index 127e70f..1824377 100644 --- a/pkg/version/version +++ b/pkg/version/version @@ -1 +1 @@ -v0.29.15 \ No newline at end of file +v0.29.16 \ No newline at end of file diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 29285e6..311c315 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -33,10 +33,11 @@ if [[ ! -x "$BENCHMARK_BIN" ]]; then echo "Building benchmark binary (pure Go + purego)..." cd "$REPO_ROOT/cmd/benchmark" CGO_ENABLED=0 go build -o "$BENCHMARK_BIN" . - # Copy libsecp256k1.so if available (runtime optional) - if [[ -f "$REPO_ROOT/pkg/crypto/p8k/libsecp256k1.so" ]]; then - cp "$REPO_ROOT/pkg/crypto/p8k/libsecp256k1.so" "$(dirname "$BENCHMARK_BIN")/" - fi + # Download libsecp256k1.so from nostr repository (runtime optional) + wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so \ + -O "$(dirname "$BENCHMARK_BIN")/libsecp256k1.so" 2>/dev/null || \ + echo "Warning: Failed to download libsecp256k1.so (optional for performance)" + chmod +x "$(dirname "$BENCHMARK_BIN")/libsecp256k1.so" 2>/dev/null || true cd "$REPO_ROOT" fi diff --git a/scripts/build-all-platforms.sh b/scripts/build-all-platforms.sh index c052299..618e015 100755 --- a/scripts/build-all-platforms.sh +++ b/scripts/build-all-platforms.sh @@ -21,7 +21,7 @@ NC='\033[0m' # No Color # Configuration VERSION=$(cat pkg/version/version) OUTPUT_DIR="$REPO_ROOT/build" -LIB_SOURCE="$REPO_ROOT/pkg/crypto/p8k" +NOSTR_REPO_BASE_URL="https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k" echo -e "${BLUE}========================================${NC}" echo -e "${BLUE}ORLY Multi-Platform Build Script${NC}" @@ -53,30 +53,38 @@ build_platform() { echo -e "${GREEN}✓ Built: ${output_name}${NC}" - # Copy appropriate runtime library + # Download appropriate runtime library from nostr repository case "$goos" in linux) - if [ -f "${LIB_SOURCE}/libsecp256k1.so" ]; then - cp "${LIB_SOURCE}/libsecp256k1.so" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so" - echo -e "${GREEN} ✓ Copied libsecp256k1.so (runtime optional)${NC}" + if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.so" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"; then + chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so" + echo -e "${GREEN} ✓ Downloaded libsecp256k1.so (runtime optional)${NC}" + else + echo -e "${YELLOW} ⚠ Failed to download libsecp256k1.so (runtime optional)${NC}" fi ;; darwin) - if [ -f "${LIB_SOURCE}/libsecp256k1.dylib" ]; then - cp "${LIB_SOURCE}/libsecp256k1.dylib" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dylib" - echo -e "${GREEN} ✓ Copied libsecp256k1.dylib (runtime optional)${NC}" + if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.dylib" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dylib"; then + chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dylib" + echo -e "${GREEN} ✓ Downloaded libsecp256k1.dylib (runtime optional)${NC}" + else + echo -e "${YELLOW} ⚠ Failed to download libsecp256k1.dylib (runtime optional)${NC}" fi ;; windows) - if [ -f "${LIB_SOURCE}/libsecp256k1.dll" ]; then - cp "${LIB_SOURCE}/libsecp256k1.dll" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dll" - echo -e "${GREEN} ✓ Copied libsecp256k1.dll (runtime optional)${NC}" + if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.dll" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dll"; then + chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dll" + echo -e "${GREEN} ✓ Downloaded libsecp256k1.dll (runtime optional)${NC}" + else + echo -e "${YELLOW} ⚠ Failed to download libsecp256k1.dll (runtime optional)${NC}" fi ;; android) - if [ -f "${LIB_SOURCE}/libsecp256k1.so" ]; then - cp "${LIB_SOURCE}/libsecp256k1.so" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so" - echo -e "${GREEN} ✓ Copied libsecp256k1.so (runtime optional)${NC}" + if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.so" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"; then + chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so" + echo -e "${GREEN} ✓ Downloaded libsecp256k1.so (runtime optional)${NC}" + else + echo -e "${YELLOW} ⚠ Failed to download libsecp256k1.so (runtime optional)${NC}" fi ;; esac diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 094e036..a89f119 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -197,10 +197,13 @@ build_application() { log_info "Building binary in current directory (pure Go + purego)..." CGO_ENABLED=0 go build -o "$BINARY_NAME" - # Copy libsecp256k1.so next to the binary (optional, for runtime performance) - if [[ -f "pkg/crypto/p8k/libsecp256k1.so" ]]; then - cp pkg/crypto/p8k/libsecp256k1.so . - log_info "Copied libsecp256k1.so next to binary (runtime optional)" + # Download libsecp256k1.so from nostr repository (optional, for runtime performance) + log_info "Downloading libsecp256k1.so from nostr repository..." + if wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so; then + chmod +x libsecp256k1.so + log_success "Downloaded libsecp256k1.so successfully (runtime optional)" + else + log_warning "Failed to download libsecp256k1.so - relay will still work but may have slower crypto" fi if [[ -f "./$BINARY_NAME" ]]; then diff --git a/scripts/runtests.sh b/scripts/runtests.sh index 8897394..82249e4 100755 --- a/scripts/runtests.sh +++ b/scripts/runtests.sh @@ -2,7 +2,16 @@ # Pure Go build with purego - no CGO needed # libsecp256k1 is loaded dynamically at runtime if available export CGO_ENABLED=0 -if [ -f "pkg/crypto/p8k/libsecp256k1.so" ]; then - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)/pkg/crypto/p8k" + +# Download libsecp256k1.so from nostr repository if not present +if [ ! -f "libsecp256k1.so" ]; then + wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so 2>/dev/null || true + chmod +x libsecp256k1.so 2>/dev/null || true fi + +# Set LD_LIBRARY_PATH if library is available +if [ -f "libsecp256k1.so" ]; then + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)" +fi + go test -v ./... -bench=. -run=xxx -benchmem \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh index af1a378..f49b245 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -2,8 +2,19 @@ # Pure Go build with purego - no CGO needed # libsecp256k1 is loaded dynamically at runtime if available export CGO_ENABLED=0 -if [ -f "pkg/crypto/p8k/libsecp256k1.so" ]; then - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)/pkg/crypto/p8k" + +# Download libsecp256k1.so from nostr repository if not present +if [ ! -f "libsecp256k1.so" ]; then + echo "Downloading libsecp256k1.so from nostr repository..." + wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so || { + echo "Warning: Failed to download libsecp256k1.so - tests may fail" + } + chmod +x libsecp256k1.so 2>/dev/null || true +fi + +# Set LD_LIBRARY_PATH to include current directory +if [ -f "libsecp256k1.so" ]; then + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)" fi go mod tidy