fix workflow to fetch libsecp256k1.so

2025-11-25 06:03:22 +00:00
parent 7fe1154391
commit 2e865c9616
12 changed files with 109 additions and 52 deletions
--- a/.claude/settings.local.json
+++ b/.claude/settings.local.json
@@ -130,7 +130,8 @@
      "Bash(sudo journalctl:*)",
      "Bash(systemctl:*)",
      "Bash(systemctl show:*)",
-      "Bash(ssh relay1:*)"
+      "Bash(ssh relay1:*)",
+      "Bash(done)"
    ],
    "deny": [],
    "ask": []
--- a/.gitea/workflows/go.yml
+++ b/.gitea/workflows/go.yml
@@ -55,8 +55,12 @@ jobs:
          export PATH=/usr/local/go/bin:$PATH
          cd ${GITHUB_WORKSPACE}
          echo "Running tests..."
-          # Copy the libsecp256k1.so to root directory so tests can find it
-          cp pkg/crypto/p8k/libsecp256k1.so .
+          # Download libsecp256k1.so from nostr repository
+          echo "Downloading libsecp256k1.so from nostr repository..."
+          wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so
+          chmod +x libsecp256k1.so
+          # Set LD_LIBRARY_PATH so tests can find the library
+          export LD_LIBRARY_PATH=${GITHUB_WORKSPACE}:${LD_LIBRARY_PATH}
          CGO_ENABLED=0 go test -v $(go list ./... | grep -v '/cmd/benchmark/external/' | xargs -n1 sh -c 'ls $0/*_test.go 1>/dev/null 2>&1 && echo $0' | grep .) || true

      - name: Build Release Binaries (Pure Go + purego)
@@ -71,8 +75,10 @@ jobs:
          # Create directory for binaries
          mkdir -p release-binaries

-          # Copy the pre-compiled libsecp256k1.so for Linux AMD64
-          cp pkg/crypto/p8k/libsecp256k1.so release-binaries/libsecp256k1-linux-amd64.so
+          # Download the pre-compiled libsecp256k1.so for Linux AMD64 from nostr repository
+          echo "Downloading libsecp256k1.so from nostr repository..."
+          wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O release-binaries/libsecp256k1-linux-amd64.so
+          chmod +x release-binaries/libsecp256k1-linux-amd64.so

          # Build for Linux AMD64 (pure Go + purego dynamic loading)
          echo "Building Linux AMD64 (pure Go + purego dynamic loading)..."
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -59,8 +59,10 @@ cd app/web && bun run dev
 # Or manually with purego setup
 CGO_ENABLED=0 go test ./...

-# Note: libsecp256k1.so must be available for crypto tests
-export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)/pkg/crypto/p8k"
+# Note: libsecp256k1.so is automatically downloaded by test.sh if needed
+# It can also be manually downloaded from the nostr repository:
+# wget https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so
+# export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)"
 ```

 ### Run Specific Package Tests
@@ -92,8 +94,8 @@ go run cmd/relay-tester/main.go -url ws://localhost:3334 -test "Basic Event"
 # Run Go benchmarks in specific package
 go test -bench=. -benchmem ./pkg/database

-# Crypto benchmarks
-cd pkg/crypto/p8k && make bench
+# Note: Crypto benchmarks are now in the external nostr library at:
+# https://git.mleku.dev/mleku/nostr

 # Run full relay benchmark suite
 cd cmd/benchmark
@@ -203,15 +205,15 @@ export ORLY_DB_INDEX_CACHE_MB=256        # Index cache size
 - `hex/` - SIMD-accelerated hex encoding using templexxx/xhex
 - `timestamp/`, `kind/`, `tag/` - Specialized field encoders

-**`pkg/crypto/`** - Cryptographic operations
- `p8k/` - Pure Go secp256k1 using purego (no CGO) to dynamically load libsecp256k1.so
-  - `secp.go` - Dynamic library loading and function binding
-  - `schnorr.go` - Schnorr signature operations (NIP-01)
-  - `ecdh.go` - ECDH for encrypted DMs (NIP-04, NIP-44)
-  - `recovery.go` - Public key recovery from signatures
-  - `libsecp256k1.so` - Pre-compiled secp256k1 library
- `keys/` - Key derivation and conversion utilities
- `sha256/` - SIMD-accelerated SHA256 using minio/sha256-simd
+**Cryptographic operations** (from `git.mleku.dev/mleku/nostr` library)
+- Pure Go secp256k1 using purego (no CGO) to dynamically load libsecp256k1.so
+  - Schnorr signature operations (NIP-01)
+  - ECDH for encrypted DMs (NIP-04, NIP-44)
+  - Public key recovery from signatures
+  - `libsecp256k1.so` - Downloaded from nostr repository at runtime/build time
+- Key derivation and conversion utilities
+- SIMD-accelerated SHA256 using minio/sha256-simd
+- SIMD-accelerated hex encoding using templexxx/xhex

 **`pkg/acl/`** - Access control systems
 - `acl.go` - ACL registry and interface
@@ -255,9 +257,11 @@ export ORLY_DB_INDEX_CACHE_MB=256        # Index cache size

 **Pure Go with Purego:**
 - All builds use `CGO_ENABLED=0`
- The p8k crypto library uses `github.com/ebitengine/purego` to dynamically load `libsecp256k1.so` at runtime
+- The p8k crypto library (from `git.mleku.dev/mleku/nostr`) uses `github.com/ebitengine/purego` to dynamically load `libsecp256k1.so` at runtime
 - This avoids CGO complexity while maintaining C library performance
- `libsecp256k1.so` must be in `LD_LIBRARY_PATH` or same directory as binary
+- `libsecp256k1.so` is automatically downloaded by build/test scripts from the nostr repository
+- Manual download: `wget https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so`
+- Library must be in `LD_LIBRARY_PATH` or same directory as binary for runtime loading

 **Database Backend Selection:**
 - Supports multiple backends via `ORLY_DB_TYPE` environment variable
--- a/7
+++ b/7
@@ -23,7 +23,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o orly -ldflags="-w -s" .
 FROM alpine:latest

 # Install runtime dependencies
-RUN apk add --no-cache ca-certificates curl
+RUN apk add --no-cache ca-certificates curl wget

 # Create app user
 RUN addgroup -g 1000 orly && \
@@ -34,7 +34,10 @@ WORKDIR /app

 # Copy binary from builder
 COPY --from=builder /build/orly /app/orly
-COPY --from=builder /build/pkg/crypto/p8k/libsecp256k1.so /app/libsecp256k1.so
+
+# Download libsecp256k1.so from nostr repository (optional for performance)
+RUN wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so \
+    -O /app/libsecp256k1.so || echo "Warning: libsecp256k1.so download failed (optional)"

 # Set library path
 ENV LD_LIBRARY_PATH=/app
--- a/docs/LIBSECP256K1_DEPLOYMENT.md
+++ b/docs/LIBSECP256K1_DEPLOYMENT.md
@@ -1,6 +1,14 @@
 # libsecp256k1 Deployment Guide

-All build scripts have been updated to ensure libsecp256k1.so is placed next to the executable.
+> **NOTE (Updated 2025):** This project now uses pure Go with purego (no CGO). The crypto library is part of the external `git.mleku.dev/mleku/nostr` dependency. The `libsecp256k1.so` file is automatically downloaded from the nostr repository during build/test. See [CLAUDE.md](../CLAUDE.md) for current build instructions.
+
+## Current Approach (Pure Go + Purego)
+
+All build scripts download `libsecp256k1.so` from `https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so` and place it next to the executable for optimal performance.
+
+## Legacy Information (For Reference)
+
+The information below describes the previous CGO-based approach and is kept for historical reference.

 ## Updated Scripts

--- a/docs/PUREGO_BUILD_SYSTEM.md
+++ b/docs/PUREGO_BUILD_SYSTEM.md
@@ -31,7 +31,7 @@ ORLY relay uses **pure Go builds (`CGO_ENABLED=0`)** across all platforms. The p

 ### Purego Dynamic Loading

-The p8k library (`pkg/crypto/p8k`) uses purego to:
+The p8k library (from `git.mleku.dev/mleku/nostr`) uses purego to:

 1. **At build time**: Compile pure Go code (`CGO_ENABLED=0`)
 2. **At runtime**: Attempt to dynamically load `libsecp256k1`
@@ -287,8 +287,11 @@ RUN go build -ldflags "-s -w" -o orly .

 # Runtime can optionally include library
 FROM alpine:latest
+RUN apk add --no-cache wget ca-certificates
 COPY --from=builder /build/orly /app/orly
-COPY --from=builder /build/pkg/crypto/p8k/libsecp256k1.so /app/ || true
+# Download libsecp256k1.so from nostr repository (optional for performance)
+RUN wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so \
+    -O /app/libsecp256k1.so || echo "Warning: libsecp256k1.so download failed (optional)"
 ENV LD_LIBRARY_PATH=/app
 CMD ["/app/orly"]
 ```
--- a/pkg/version/version
+++ b/pkg/version/version
@@ -1 +1 @@
-v0.29.15
+v0.29.16
--- a/scripts/benchmark.sh
+++ b/scripts/benchmark.sh
@@ -33,10 +33,11 @@ if [[ ! -x "$BENCHMARK_BIN" ]]; then
    echo "Building benchmark binary (pure Go + purego)..."
    cd "$REPO_ROOT/cmd/benchmark"
    CGO_ENABLED=0 go build -o "$BENCHMARK_BIN" .
-    # Copy libsecp256k1.so if available (runtime optional)
-    if [[ -f "$REPO_ROOT/pkg/crypto/p8k/libsecp256k1.so" ]]; then
-        cp "$REPO_ROOT/pkg/crypto/p8k/libsecp256k1.so" "$(dirname "$BENCHMARK_BIN")/"
-    fi
+    # Download libsecp256k1.so from nostr repository (runtime optional)
+    wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so \
+        -O "$(dirname "$BENCHMARK_BIN")/libsecp256k1.so" 2>/dev/null || \
+        echo "Warning: Failed to download libsecp256k1.so (optional for performance)"
+    chmod +x "$(dirname "$BENCHMARK_BIN")/libsecp256k1.so" 2>/dev/null || true
    cd "$REPO_ROOT"
 fi

--- a/scripts/build-all-platforms.sh
+++ b/scripts/build-all-platforms.sh
@@ -21,7 +21,7 @@ NC='\033[0m' # No Color
 # Configuration
 VERSION=$(cat pkg/version/version)
 OUTPUT_DIR="$REPO_ROOT/build"
-LIB_SOURCE="$REPO_ROOT/pkg/crypto/p8k"
+NOSTR_REPO_BASE_URL="https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k"

 echo -e "${BLUE}========================================${NC}"
 echo -e "${BLUE}ORLY Multi-Platform Build Script${NC}"
@@ -53,30 +53,38 @@ build_platform() {
        
        echo -e "${GREEN}✓ Built: ${output_name}${NC}"
        
-        # Copy appropriate runtime library
+        # Download appropriate runtime library from nostr repository
        case "$goos" in
            linux)
-                if [ -f "${LIB_SOURCE}/libsecp256k1.so" ]; then
-                    cp "${LIB_SOURCE}/libsecp256k1.so" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"
-                    echo -e "${GREEN}  ✓ Copied libsecp256k1.so (runtime optional)${NC}"
+                if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.so" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"; then
+                    chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"
+                    echo -e "${GREEN}  ✓ Downloaded libsecp256k1.so (runtime optional)${NC}"
+                else
+                    echo -e "${YELLOW}  ⚠ Failed to download libsecp256k1.so (runtime optional)${NC}"
                fi
                ;;
            darwin)
-                if [ -f "${LIB_SOURCE}/libsecp256k1.dylib" ]; then
-                    cp "${LIB_SOURCE}/libsecp256k1.dylib" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dylib"
-                    echo -e "${GREEN}  ✓ Copied libsecp256k1.dylib (runtime optional)${NC}"
+                if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.dylib" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dylib"; then
+                    chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dylib"
+                    echo -e "${GREEN}  ✓ Downloaded libsecp256k1.dylib (runtime optional)${NC}"
+                else
+                    echo -e "${YELLOW}  ⚠ Failed to download libsecp256k1.dylib (runtime optional)${NC}"
                fi
                ;;
            windows)
-                if [ -f "${LIB_SOURCE}/libsecp256k1.dll" ]; then
-                    cp "${LIB_SOURCE}/libsecp256k1.dll" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dll"
-                    echo -e "${GREEN}  ✓ Copied libsecp256k1.dll (runtime optional)${NC}"
+                if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.dll" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dll"; then
+                    chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.dll"
+                    echo -e "${GREEN}  ✓ Downloaded libsecp256k1.dll (runtime optional)${NC}"
+                else
+                    echo -e "${YELLOW}  ⚠ Failed to download libsecp256k1.dll (runtime optional)${NC}"
                fi
                ;;
            android)
-                if [ -f "${LIB_SOURCE}/libsecp256k1.so" ]; then
-                    cp "${LIB_SOURCE}/libsecp256k1.so" "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"
-                    echo -e "${GREEN}  ✓ Copied libsecp256k1.so (runtime optional)${NC}"
+                if wget -q "${NOSTR_REPO_BASE_URL}/libsecp256k1.so" -O "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"; then
+                    chmod +x "${OUTPUT_DIR}/libsecp256k1-${platform_name}.so"
+                    echo -e "${GREEN}  ✓ Downloaded libsecp256k1.so (runtime optional)${NC}"
+                else
+                    echo -e "${YELLOW}  ⚠ Failed to download libsecp256k1.so (runtime optional)${NC}"
                fi
                ;;
        esac
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -197,10 +197,13 @@ build_application() {
    log_info "Building binary in current directory (pure Go + purego)..."
    CGO_ENABLED=0 go build -o "$BINARY_NAME"
    
-    # Copy libsecp256k1.so next to the binary (optional, for runtime performance)
-    if [[ -f "pkg/crypto/p8k/libsecp256k1.so" ]]; then
-        cp pkg/crypto/p8k/libsecp256k1.so .
-        log_info "Copied libsecp256k1.so next to binary (runtime optional)"
+    # Download libsecp256k1.so from nostr repository (optional, for runtime performance)
+    log_info "Downloading libsecp256k1.so from nostr repository..."
+    if wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so; then
+        chmod +x libsecp256k1.so
+        log_success "Downloaded libsecp256k1.so successfully (runtime optional)"
+    else
+        log_warning "Failed to download libsecp256k1.so - relay will still work but may have slower crypto"
    fi
    
    if [[ -f "./$BINARY_NAME" ]]; then
--- a/scripts/runtests.sh
+++ b/scripts/runtests.sh
@@ -2,7 +2,16 @@
 # Pure Go build with purego - no CGO needed
 # libsecp256k1 is loaded dynamically at runtime if available
 export CGO_ENABLED=0
-if [ -f "pkg/crypto/p8k/libsecp256k1.so" ]; then
-    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)/pkg/crypto/p8k"
+
+# Download libsecp256k1.so from nostr repository if not present
+if [ ! -f "libsecp256k1.so" ]; then
+    wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so 2>/dev/null || true
+    chmod +x libsecp256k1.so 2>/dev/null || true
 fi
+
+# Set LD_LIBRARY_PATH if library is available
+if [ -f "libsecp256k1.so" ]; then
+    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)"
+fi
+
 go test -v ./... -bench=. -run=xxx -benchmem
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -2,8 +2,19 @@
 # Pure Go build with purego - no CGO needed
 # libsecp256k1 is loaded dynamically at runtime if available
 export CGO_ENABLED=0
-if [ -f "pkg/crypto/p8k/libsecp256k1.so" ]; then
-    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)/pkg/crypto/p8k"
+
+# Download libsecp256k1.so from nostr repository if not present
+if [ ! -f "libsecp256k1.so" ]; then
+    echo "Downloading libsecp256k1.so from nostr repository..."
+    wget -q https://git.mleku.dev/mleku/nostr/raw/branch/main/crypto/p8k/libsecp256k1.so -O libsecp256k1.so || {
+        echo "Warning: Failed to download libsecp256k1.so - tests may fail"
+    }
+    chmod +x libsecp256k1.so 2>/dev/null || true
+fi
+
+# Set LD_LIBRARY_PATH to include current directory
+if [ -f "libsecp256k1.so" ]; then
+    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(pwd)"
 fi

 go mod tidy