Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
6c7d55ff7e
|
|||
|
3c17e975df
|
|||
|
feae79af1a
|
|||
|
ebef8605eb
|
|||
|
c5db0abf73
|
@@ -153,7 +153,9 @@
|
||||
"Bash(git check-ignore:*)",
|
||||
"Bash(git commit:*)",
|
||||
"WebFetch(domain:www.npmjs.com)",
|
||||
"Bash(git stash:*)"
|
||||
"Bash(git stash:*)",
|
||||
"WebFetch(domain:arxiv.org)",
|
||||
"WebFetch(domain:hal.science)"
|
||||
],
|
||||
"deny": [],
|
||||
"ask": []
|
||||
|
||||
1115
.claude/skills/distributed-systems/SKILL.md
Normal file
1115
.claude/skills/distributed-systems/SKILL.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,610 @@
|
||||
# Consensus Protocols - Detailed Reference
|
||||
|
||||
Complete specifications and implementation details for major consensus protocols.
|
||||
|
||||
## Paxos Complete Specification
|
||||
|
||||
### Proposal Numbers
|
||||
|
||||
Proposal numbers must be:
|
||||
- **Unique**: No two proposers use the same number
|
||||
- **Totally ordered**: Any two can be compared
|
||||
|
||||
**Implementation**: `(round_number, proposer_id)` where proposer_id breaks ties.
|
||||
|
||||
### Single-Decree Paxos State
|
||||
|
||||
**Proposer state**:
|
||||
```
|
||||
proposal_number: int
|
||||
value: any
|
||||
```
|
||||
|
||||
**Acceptor state (persistent)**:
|
||||
```
|
||||
highest_promised: int # Highest proposal number promised
|
||||
accepted_proposal: int # Number of accepted proposal (0 if none)
|
||||
accepted_value: any # Value of accepted proposal (null if none)
|
||||
```
|
||||
|
||||
### Message Format
|
||||
|
||||
**Prepare** (Phase 1a):
|
||||
```
|
||||
{
|
||||
type: "PREPARE",
|
||||
proposal_number: n
|
||||
}
|
||||
```
|
||||
|
||||
**Promise** (Phase 1b):
|
||||
```
|
||||
{
|
||||
type: "PROMISE",
|
||||
proposal_number: n,
|
||||
accepted_proposal: m, # null if nothing accepted
|
||||
accepted_value: v # null if nothing accepted
|
||||
}
|
||||
```
|
||||
|
||||
**Accept** (Phase 2a):
|
||||
```
|
||||
{
|
||||
type: "ACCEPT",
|
||||
proposal_number: n,
|
||||
value: v
|
||||
}
|
||||
```
|
||||
|
||||
**Accepted** (Phase 2b):
|
||||
```
|
||||
{
|
||||
type: "ACCEPTED",
|
||||
proposal_number: n,
|
||||
value: v
|
||||
}
|
||||
```
|
||||
|
||||
### Proposer Algorithm
|
||||
|
||||
```
|
||||
function propose(value):
|
||||
n = generate_proposal_number()
|
||||
|
||||
# Phase 1: Prepare
|
||||
promises = []
|
||||
for acceptor in acceptors:
|
||||
send PREPARE(n) to acceptor
|
||||
|
||||
wait until |promises| > |acceptors|/2 or timeout
|
||||
|
||||
if timeout:
|
||||
return FAILED
|
||||
|
||||
# Choose value
|
||||
highest = max(promises, key=p.accepted_proposal)
|
||||
if highest.accepted_value is not null:
|
||||
value = highest.accepted_value
|
||||
|
||||
# Phase 2: Accept
|
||||
accepts = []
|
||||
for acceptor in acceptors:
|
||||
send ACCEPT(n, value) to acceptor
|
||||
|
||||
wait until |accepts| > |acceptors|/2 or timeout
|
||||
|
||||
if timeout:
|
||||
return FAILED
|
||||
|
||||
return SUCCESS(value)
|
||||
```
|
||||
|
||||
### Acceptor Algorithm
|
||||
|
||||
```
|
||||
on receive PREPARE(n):
|
||||
if n > highest_promised:
|
||||
highest_promised = n
|
||||
persist(highest_promised)
|
||||
reply PROMISE(n, accepted_proposal, accepted_value)
|
||||
else:
|
||||
# Optionally reply NACK(highest_promised)
|
||||
ignore or reject
|
||||
|
||||
on receive ACCEPT(n, v):
|
||||
if n >= highest_promised:
|
||||
highest_promised = n
|
||||
accepted_proposal = n
|
||||
accepted_value = v
|
||||
persist(highest_promised, accepted_proposal, accepted_value)
|
||||
reply ACCEPTED(n, v)
|
||||
else:
|
||||
ignore or reject
|
||||
```
|
||||
|
||||
### Multi-Paxos Optimization
|
||||
|
||||
**Stable leader**:
|
||||
```
|
||||
# Leader election (using Paxos or other method)
|
||||
leader = elect_leader()
|
||||
|
||||
# Leader's Phase 1 for all future instances
|
||||
leader sends PREPARE(n) for instance range [i, ∞)
|
||||
|
||||
# For each command:
|
||||
function propose_as_leader(value, instance):
|
||||
# Skip Phase 1 if already leader
|
||||
for acceptor in acceptors:
|
||||
send ACCEPT(n, value, instance) to acceptor
|
||||
wait for majority ACCEPTED
|
||||
return SUCCESS
|
||||
```
|
||||
|
||||
### Paxos Safety Proof Sketch
|
||||
|
||||
**Invariant**: If a value v is chosen for instance i, no other value can be chosen.
|
||||
|
||||
**Proof**:
|
||||
1. Value chosen → accepted by majority with proposal n
|
||||
2. Any higher proposal n' must contact majority
|
||||
3. Majorities intersect → at least one acceptor has accepted v
|
||||
4. New proposer adopts v (or higher already-accepted value)
|
||||
5. By induction, all future proposals use v
|
||||
|
||||
## Raft Complete Specification
|
||||
|
||||
### State
|
||||
|
||||
**All servers (persistent)**:
|
||||
```
|
||||
currentTerm: int # Latest term seen
|
||||
votedFor: ServerId # Candidate voted for in current term (null if none)
|
||||
log[]: LogEntry # Log entries
|
||||
```
|
||||
|
||||
**All servers (volatile)**:
|
||||
```
|
||||
commitIndex: int # Highest log index known to be committed
|
||||
lastApplied: int # Highest log index applied to state machine
|
||||
```
|
||||
|
||||
**Leader (volatile, reinitialized after election)**:
|
||||
```
|
||||
nextIndex[]: int # For each server, next log index to send
|
||||
matchIndex[]: int # For each server, highest log index replicated
|
||||
```
|
||||
|
||||
**LogEntry**:
|
||||
```
|
||||
{
|
||||
term: int,
|
||||
command: any
|
||||
}
|
||||
```
|
||||
|
||||
### RequestVote RPC
|
||||
|
||||
**Request**:
|
||||
```
|
||||
{
|
||||
term: int, # Candidate's term
|
||||
candidateId: ServerId, # Candidate requesting vote
|
||||
lastLogIndex: int, # Index of candidate's last log entry
|
||||
lastLogTerm: int # Term of candidate's last log entry
|
||||
}
|
||||
```
|
||||
|
||||
**Response**:
|
||||
```
|
||||
{
|
||||
term: int, # currentTerm, for candidate to update itself
|
||||
voteGranted: bool # True if candidate received vote
|
||||
}
|
||||
```
|
||||
|
||||
**Receiver implementation**:
|
||||
```
|
||||
on receive RequestVote(term, candidateId, lastLogIndex, lastLogTerm):
|
||||
if term < currentTerm:
|
||||
return {term: currentTerm, voteGranted: false}
|
||||
|
||||
if term > currentTerm:
|
||||
currentTerm = term
|
||||
votedFor = null
|
||||
convert to follower
|
||||
|
||||
# Check if candidate's log is at least as up-to-date as ours
|
||||
ourLastTerm = log[len(log)-1].term if log else 0
|
||||
ourLastIndex = len(log) - 1
|
||||
|
||||
logOK = (lastLogTerm > ourLastTerm) or
|
||||
(lastLogTerm == ourLastTerm and lastLogIndex >= ourLastIndex)
|
||||
|
||||
if (votedFor is null or votedFor == candidateId) and logOK:
|
||||
votedFor = candidateId
|
||||
persist(currentTerm, votedFor)
|
||||
reset election timer
|
||||
return {term: currentTerm, voteGranted: true}
|
||||
|
||||
return {term: currentTerm, voteGranted: false}
|
||||
```
|
||||
|
||||
### AppendEntries RPC
|
||||
|
||||
**Request**:
|
||||
```
|
||||
{
|
||||
term: int, # Leader's term
|
||||
leaderId: ServerId, # For follower to redirect clients
|
||||
prevLogIndex: int, # Index of log entry preceding new ones
|
||||
prevLogTerm: int, # Term of prevLogIndex entry
|
||||
entries[]: LogEntry, # Log entries to store (empty for heartbeat)
|
||||
leaderCommit: int # Leader's commitIndex
|
||||
}
|
||||
```
|
||||
|
||||
**Response**:
|
||||
```
|
||||
{
|
||||
term: int, # currentTerm, for leader to update itself
|
||||
success: bool # True if follower had matching prevLog entry
|
||||
}
|
||||
```
|
||||
|
||||
**Receiver implementation**:
|
||||
```
|
||||
on receive AppendEntries(term, leaderId, prevLogIndex, prevLogTerm, entries, leaderCommit):
|
||||
if term < currentTerm:
|
||||
return {term: currentTerm, success: false}
|
||||
|
||||
reset election timer
|
||||
|
||||
if term > currentTerm:
|
||||
currentTerm = term
|
||||
votedFor = null
|
||||
|
||||
convert to follower
|
||||
|
||||
# Check log consistency
|
||||
if prevLogIndex >= len(log) or
|
||||
(prevLogIndex >= 0 and log[prevLogIndex].term != prevLogTerm):
|
||||
return {term: currentTerm, success: false}
|
||||
|
||||
# Append new entries (handling conflicts)
|
||||
for i, entry in enumerate(entries):
|
||||
index = prevLogIndex + 1 + i
|
||||
if index < len(log):
|
||||
if log[index].term != entry.term:
|
||||
# Delete conflicting entry and all following
|
||||
log = log[:index]
|
||||
log.append(entry)
|
||||
else:
|
||||
log.append(entry)
|
||||
|
||||
persist(currentTerm, votedFor, log)
|
||||
|
||||
# Update commit index
|
||||
if leaderCommit > commitIndex:
|
||||
commitIndex = min(leaderCommit, len(log) - 1)
|
||||
|
||||
return {term: currentTerm, success: true}
|
||||
```
|
||||
|
||||
### Leader Behavior
|
||||
|
||||
```
|
||||
on becoming leader:
|
||||
for each server:
|
||||
nextIndex[server] = len(log)
|
||||
matchIndex[server] = 0
|
||||
|
||||
start sending heartbeats
|
||||
|
||||
on receiving client command:
|
||||
append entry to local log
|
||||
persist log
|
||||
send AppendEntries to all followers
|
||||
|
||||
on receiving AppendEntries response from server:
|
||||
if response.success:
|
||||
matchIndex[server] = prevLogIndex + len(entries)
|
||||
nextIndex[server] = matchIndex[server] + 1
|
||||
|
||||
# Update commit index
|
||||
for N from commitIndex+1 to len(log)-1:
|
||||
if log[N].term == currentTerm and
|
||||
|{s : matchIndex[s] >= N}| > |servers|/2:
|
||||
commitIndex = N
|
||||
else:
|
||||
nextIndex[server] = max(1, nextIndex[server] - 1)
|
||||
retry AppendEntries with lower prevLogIndex
|
||||
|
||||
on commitIndex update:
|
||||
while lastApplied < commitIndex:
|
||||
lastApplied++
|
||||
apply log[lastApplied].command to state machine
|
||||
```
|
||||
|
||||
### Election Timeout
|
||||
|
||||
```
|
||||
on election timeout (follower or candidate):
|
||||
currentTerm++
|
||||
convert to candidate
|
||||
votedFor = self
|
||||
persist(currentTerm, votedFor)
|
||||
reset election timer
|
||||
votes = 1 # Vote for self
|
||||
|
||||
for each server except self:
|
||||
send RequestVote(currentTerm, self, lastLogIndex, lastLogTerm)
|
||||
|
||||
wait for responses or timeout:
|
||||
if received votes > |servers|/2:
|
||||
become leader
|
||||
if received AppendEntries from valid leader:
|
||||
become follower
|
||||
if timeout:
|
||||
start new election
|
||||
```
|
||||
|
||||
## PBFT Complete Specification
|
||||
|
||||
### Message Types
|
||||
|
||||
**REQUEST**:
|
||||
```
|
||||
{
|
||||
type: "REQUEST",
|
||||
operation: o, # Operation to execute
|
||||
timestamp: t, # Client timestamp (for reply matching)
|
||||
client: c # Client identifier
|
||||
}
|
||||
```
|
||||
|
||||
**PRE-PREPARE**:
|
||||
```
|
||||
{
|
||||
type: "PRE-PREPARE",
|
||||
view: v, # Current view number
|
||||
sequence: n, # Sequence number
|
||||
digest: d, # Hash of request
|
||||
request: m # The request message
|
||||
}
|
||||
signature(primary)
|
||||
```
|
||||
|
||||
**PREPARE**:
|
||||
```
|
||||
{
|
||||
type: "PREPARE",
|
||||
view: v,
|
||||
sequence: n,
|
||||
digest: d,
|
||||
replica: i # Sending replica
|
||||
}
|
||||
signature(replica_i)
|
||||
```
|
||||
|
||||
**COMMIT**:
|
||||
```
|
||||
{
|
||||
type: "COMMIT",
|
||||
view: v,
|
||||
sequence: n,
|
||||
digest: d,
|
||||
replica: i
|
||||
}
|
||||
signature(replica_i)
|
||||
```
|
||||
|
||||
**REPLY**:
|
||||
```
|
||||
{
|
||||
type: "REPLY",
|
||||
view: v,
|
||||
timestamp: t,
|
||||
client: c,
|
||||
replica: i,
|
||||
result: r # Execution result
|
||||
}
|
||||
signature(replica_i)
|
||||
```
|
||||
|
||||
### Replica State
|
||||
|
||||
```
|
||||
view: int # Current view
|
||||
sequence: int # Last assigned sequence number (primary)
|
||||
log[]: {request, prepares, commits, state} # Log of requests
|
||||
prepared_certificates: {} # Prepared certificates (2f+1 prepares)
|
||||
committed_certificates: {} # Committed certificates (2f+1 commits)
|
||||
h: int # Low water mark
|
||||
H: int # High water mark (h + L)
|
||||
```
|
||||
|
||||
### Normal Operation Protocol
|
||||
|
||||
**Primary (replica p = v mod n)**:
|
||||
```
|
||||
on receive REQUEST(m) from client:
|
||||
if not primary for current view:
|
||||
forward to primary
|
||||
return
|
||||
|
||||
n = assign_sequence_number()
|
||||
d = hash(m)
|
||||
|
||||
broadcast PRE-PREPARE(v, n, d, m) to all replicas
|
||||
add to log
|
||||
```
|
||||
|
||||
**All replicas**:
|
||||
```
|
||||
on receive PRE-PREPARE(v, n, d, m) from primary:
|
||||
if v != current_view:
|
||||
ignore
|
||||
if already accepted pre-prepare for (v, n) with different digest:
|
||||
ignore
|
||||
if not in_view_as_backup(v):
|
||||
ignore
|
||||
if not h < n <= H:
|
||||
ignore # Outside sequence window
|
||||
|
||||
# Valid pre-prepare
|
||||
add to log
|
||||
broadcast PREPARE(v, n, d, i) to all replicas
|
||||
|
||||
on receive PREPARE(v, n, d, j) from replica j:
|
||||
if v != current_view:
|
||||
ignore
|
||||
|
||||
add to log[n].prepares
|
||||
|
||||
if |log[n].prepares| >= 2f and not already_prepared(v, n, d):
|
||||
# Prepared certificate complete
|
||||
mark as prepared
|
||||
broadcast COMMIT(v, n, d, i) to all replicas
|
||||
|
||||
on receive COMMIT(v, n, d, j) from replica j:
|
||||
if v != current_view:
|
||||
ignore
|
||||
|
||||
add to log[n].commits
|
||||
|
||||
if |log[n].commits| >= 2f + 1 and prepared(v, n, d):
|
||||
# Committed certificate complete
|
||||
if all entries < n are committed:
|
||||
execute(m)
|
||||
send REPLY(v, t, c, i, result) to client
|
||||
```
|
||||
|
||||
### View Change Protocol
|
||||
|
||||
**Timeout trigger**:
|
||||
```
|
||||
on request timeout (no progress):
|
||||
view_change_timeout++
|
||||
broadcast VIEW-CHANGE(v+1, n, C, P, i)
|
||||
|
||||
where:
|
||||
n = last stable checkpoint sequence number
|
||||
C = checkpoint certificate (2f+1 checkpoint messages)
|
||||
P = set of prepared certificates for messages after n
|
||||
```
|
||||
|
||||
**VIEW-CHANGE**:
|
||||
```
|
||||
{
|
||||
type: "VIEW-CHANGE",
|
||||
view: v, # New view number
|
||||
sequence: n, # Checkpoint sequence
|
||||
checkpoints: C, # Checkpoint certificate
|
||||
prepared: P, # Set of prepared certificates
|
||||
replica: i
|
||||
}
|
||||
signature(replica_i)
|
||||
```
|
||||
|
||||
**New primary (p' = v mod n)**:
|
||||
```
|
||||
on receive 2f VIEW-CHANGE for view v:
|
||||
V = set of valid view-change messages
|
||||
|
||||
# Compute O: set of requests to re-propose
|
||||
O = {}
|
||||
for seq in max_checkpoint_seq(V) to max_seq(V):
|
||||
if exists prepared certificate for seq in V:
|
||||
O[seq] = request from certificate
|
||||
else:
|
||||
O[seq] = null-request # No-op
|
||||
|
||||
broadcast NEW-VIEW(v, V, O)
|
||||
|
||||
# Re-run protocol for requests in O
|
||||
for seq, request in O:
|
||||
if request != null:
|
||||
send PRE-PREPARE(v, seq, hash(request), request)
|
||||
```
|
||||
|
||||
**NEW-VIEW**:
|
||||
```
|
||||
{
|
||||
type: "NEW-VIEW",
|
||||
view: v,
|
||||
view_changes: V, # 2f+1 view-change messages
|
||||
pre_prepares: O # Set of pre-prepare messages
|
||||
}
|
||||
signature(primary)
|
||||
```
|
||||
|
||||
### Checkpointing
|
||||
|
||||
Periodic stable checkpoints to garbage collect logs:
|
||||
|
||||
```
|
||||
every K requests:
|
||||
state_hash = hash(state_machine_state)
|
||||
broadcast CHECKPOINT(n, state_hash, i)
|
||||
|
||||
on receive 2f+1 CHECKPOINT for (n, d):
|
||||
if all digests match:
|
||||
create stable checkpoint
|
||||
h = n # Move low water mark
|
||||
garbage_collect(entries < n)
|
||||
```
|
||||
|
||||
## HotStuff Protocol
|
||||
|
||||
Linear complexity BFT using threshold signatures.
|
||||
|
||||
### Key Innovation
|
||||
|
||||
- **Three-phase**: prepare → pre-commit → commit → decide
|
||||
- **Pipelining**: Next proposal starts before current finishes
|
||||
- **Threshold signatures**: O(n) total messages instead of O(n²)
|
||||
|
||||
### Message Flow
|
||||
|
||||
```
|
||||
Phase 1 (Prepare):
|
||||
Leader: broadcast PREPARE(v, node)
|
||||
Replicas: sign and send partial signature to leader
|
||||
Leader: aggregate into prepare certificate QC
|
||||
|
||||
Phase 2 (Pre-commit):
|
||||
Leader: broadcast PRE-COMMIT(v, QC_prepare)
|
||||
Replicas: sign and send partial signature
|
||||
Leader: aggregate into pre-commit certificate
|
||||
|
||||
Phase 3 (Commit):
|
||||
Leader: broadcast COMMIT(v, QC_precommit)
|
||||
Replicas: sign and send partial signature
|
||||
Leader: aggregate into commit certificate
|
||||
|
||||
Phase 4 (Decide):
|
||||
Leader: broadcast DECIDE(v, QC_commit)
|
||||
Replicas: execute and commit
|
||||
```
|
||||
|
||||
### Pipelining
|
||||
|
||||
```
|
||||
Block k: [prepare] [pre-commit] [commit] [decide]
|
||||
Block k+1: [prepare] [pre-commit] [commit] [decide]
|
||||
Block k+2: [prepare] [pre-commit] [commit] [decide]
|
||||
```
|
||||
|
||||
Each phase of block k+1 piggybacks on messages for block k.
|
||||
|
||||
## Protocol Comparison Matrix
|
||||
|
||||
| Feature | Paxos | Raft | PBFT | HotStuff |
|
||||
|---------|-------|------|------|----------|
|
||||
| Fault model | Crash | Crash | Byzantine | Byzantine |
|
||||
| Fault tolerance | f with 2f+1 | f with 2f+1 | f with 3f+1 | f with 3f+1 |
|
||||
| Message complexity | O(n) | O(n) | O(n²) | O(n) |
|
||||
| Leader required | No (helps) | Yes | Yes | Yes |
|
||||
| Phases | 2 | 2 | 3 | 3 |
|
||||
| View change | Complex | Simple | Complex | Simple |
|
||||
610
.claude/skills/distributed-systems/references/logical-clocks.md
Normal file
610
.claude/skills/distributed-systems/references/logical-clocks.md
Normal file
@@ -0,0 +1,610 @@
|
||||
# Logical Clocks - Implementation Reference
|
||||
|
||||
Detailed implementations and algorithms for causality tracking.
|
||||
|
||||
## Lamport Clock Implementation
|
||||
|
||||
### Data Structure
|
||||
|
||||
```go
|
||||
type LamportClock struct {
|
||||
counter uint64
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewLamportClock() *LamportClock {
|
||||
return &LamportClock{counter: 0}
|
||||
}
|
||||
```
|
||||
|
||||
### Operations
|
||||
|
||||
```go
|
||||
// Tick increments clock for local event
|
||||
func (c *LamportClock) Tick() uint64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.counter++
|
||||
return c.counter
|
||||
}
|
||||
|
||||
// Send returns timestamp for outgoing message
|
||||
func (c *LamportClock) Send() uint64 {
|
||||
return c.Tick()
|
||||
}
|
||||
|
||||
// Receive updates clock based on incoming message timestamp
|
||||
func (c *LamportClock) Receive(msgTime uint64) uint64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if msgTime > c.counter {
|
||||
c.counter = msgTime
|
||||
}
|
||||
c.counter++
|
||||
return c.counter
|
||||
}
|
||||
|
||||
// Time returns current clock value without incrementing
|
||||
func (c *LamportClock) Time() uint64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.counter
|
||||
}
|
||||
```
|
||||
|
||||
### Usage Example
|
||||
|
||||
```go
|
||||
// Process A
|
||||
clockA := NewLamportClock()
|
||||
e1 := clockA.Tick() // Event 1: time=1
|
||||
msgTime := clockA.Send() // Send: time=2
|
||||
|
||||
// Process B
|
||||
clockB := NewLamportClock()
|
||||
e2 := clockB.Tick() // Event 2: time=1
|
||||
e3 := clockB.Receive(msgTime) // Receive: time=3 (max(1,2)+1)
|
||||
```
|
||||
|
||||
## Vector Clock Implementation
|
||||
|
||||
### Data Structure
|
||||
|
||||
```go
|
||||
type VectorClock struct {
|
||||
clocks map[string]uint64 // processID -> logical time
|
||||
self string // this process's ID
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func NewVectorClock(processID string, allProcesses []string) *VectorClock {
|
||||
clocks := make(map[string]uint64)
|
||||
for _, p := range allProcesses {
|
||||
clocks[p] = 0
|
||||
}
|
||||
return &VectorClock{
|
||||
clocks: clocks,
|
||||
self: processID,
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Operations
|
||||
|
||||
```go
|
||||
// Tick increments own clock
|
||||
func (vc *VectorClock) Tick() map[string]uint64 {
|
||||
vc.mu.Lock()
|
||||
defer vc.mu.Unlock()
|
||||
|
||||
vc.clocks[vc.self]++
|
||||
return vc.copy()
|
||||
}
|
||||
|
||||
// Send returns copy of vector for message
|
||||
func (vc *VectorClock) Send() map[string]uint64 {
|
||||
return vc.Tick()
|
||||
}
|
||||
|
||||
// Receive merges incoming vector and increments
|
||||
func (vc *VectorClock) Receive(incoming map[string]uint64) map[string]uint64 {
|
||||
vc.mu.Lock()
|
||||
defer vc.mu.Unlock()
|
||||
|
||||
// Merge: take max of each component
|
||||
for pid, time := range incoming {
|
||||
if time > vc.clocks[pid] {
|
||||
vc.clocks[pid] = time
|
||||
}
|
||||
}
|
||||
|
||||
// Increment own clock
|
||||
vc.clocks[vc.self]++
|
||||
return vc.copy()
|
||||
}
|
||||
|
||||
// copy returns a copy of the vector
|
||||
func (vc *VectorClock) copy() map[string]uint64 {
|
||||
result := make(map[string]uint64)
|
||||
for k, v := range vc.clocks {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
```
|
||||
|
||||
### Comparison Functions
|
||||
|
||||
```go
|
||||
// Compare returns ordering relationship between two vectors
|
||||
type Ordering int
|
||||
|
||||
const (
|
||||
Equal Ordering = iota // V1 == V2
|
||||
HappenedBefore // V1 < V2
|
||||
HappenedAfter // V1 > V2
|
||||
Concurrent // V1 || V2
|
||||
)
|
||||
|
||||
func Compare(v1, v2 map[string]uint64) Ordering {
|
||||
less := false
|
||||
greater := false
|
||||
|
||||
// Get all keys
|
||||
allKeys := make(map[string]bool)
|
||||
for k := range v1 {
|
||||
allKeys[k] = true
|
||||
}
|
||||
for k := range v2 {
|
||||
allKeys[k] = true
|
||||
}
|
||||
|
||||
for k := range allKeys {
|
||||
t1 := v1[k] // 0 if not present
|
||||
t2 := v2[k]
|
||||
|
||||
if t1 < t2 {
|
||||
less = true
|
||||
}
|
||||
if t1 > t2 {
|
||||
greater = true
|
||||
}
|
||||
}
|
||||
|
||||
if !less && !greater {
|
||||
return Equal
|
||||
}
|
||||
if less && !greater {
|
||||
return HappenedBefore
|
||||
}
|
||||
if greater && !less {
|
||||
return HappenedAfter
|
||||
}
|
||||
return Concurrent
|
||||
}
|
||||
|
||||
// IsConcurrent checks if two events are concurrent
|
||||
func IsConcurrent(v1, v2 map[string]uint64) bool {
|
||||
return Compare(v1, v2) == Concurrent
|
||||
}
|
||||
|
||||
// HappenedBefore checks if v1 -> v2 (v1 causally precedes v2)
|
||||
func HappenedBefore(v1, v2 map[string]uint64) bool {
|
||||
return Compare(v1, v2) == HappenedBefore
|
||||
}
|
||||
```
|
||||
|
||||
## Interval Tree Clock Implementation
|
||||
|
||||
### Data Structures
|
||||
|
||||
```go
|
||||
// ID represents the identity tree
|
||||
type ID struct {
|
||||
IsLeaf bool
|
||||
Value int // 0 or 1 for leaves
|
||||
Left *ID // nil for leaves
|
||||
Right *ID
|
||||
}
|
||||
|
||||
// Stamp represents the event tree
|
||||
type Stamp struct {
|
||||
Base int
|
||||
Left *Stamp // nil for leaf stamps
|
||||
Right *Stamp
|
||||
}
|
||||
|
||||
// ITC combines ID and Stamp
|
||||
type ITC struct {
|
||||
ID *ID
|
||||
Stamp *Stamp
|
||||
}
|
||||
```
|
||||
|
||||
### ID Operations
|
||||
|
||||
```go
|
||||
// NewSeedID creates initial full ID (1)
|
||||
func NewSeedID() *ID {
|
||||
return &ID{IsLeaf: true, Value: 1}
|
||||
}
|
||||
|
||||
// Fork splits an ID into two
|
||||
func (id *ID) Fork() (*ID, *ID) {
|
||||
if id.IsLeaf {
|
||||
if id.Value == 0 {
|
||||
// Cannot fork zero ID
|
||||
return &ID{IsLeaf: true, Value: 0},
|
||||
&ID{IsLeaf: true, Value: 0}
|
||||
}
|
||||
// Split full ID into left and right halves
|
||||
return &ID{
|
||||
IsLeaf: false,
|
||||
Left: &ID{IsLeaf: true, Value: 1},
|
||||
Right: &ID{IsLeaf: true, Value: 0},
|
||||
},
|
||||
&ID{
|
||||
IsLeaf: false,
|
||||
Left: &ID{IsLeaf: true, Value: 0},
|
||||
Right: &ID{IsLeaf: true, Value: 1},
|
||||
}
|
||||
}
|
||||
|
||||
// Fork from non-leaf: give half to each
|
||||
if id.Left.IsLeaf && id.Left.Value == 0 {
|
||||
// Left is zero, fork right
|
||||
newRight1, newRight2 := id.Right.Fork()
|
||||
return &ID{IsLeaf: false, Left: id.Left, Right: newRight1},
|
||||
&ID{IsLeaf: false, Left: &ID{IsLeaf: true, Value: 0}, Right: newRight2}
|
||||
}
|
||||
if id.Right.IsLeaf && id.Right.Value == 0 {
|
||||
// Right is zero, fork left
|
||||
newLeft1, newLeft2 := id.Left.Fork()
|
||||
return &ID{IsLeaf: false, Left: newLeft1, Right: id.Right},
|
||||
&ID{IsLeaf: false, Left: newLeft2, Right: &ID{IsLeaf: true, Value: 0}}
|
||||
}
|
||||
|
||||
// Both have IDs, split
|
||||
return &ID{IsLeaf: false, Left: id.Left, Right: &ID{IsLeaf: true, Value: 0}},
|
||||
&ID{IsLeaf: false, Left: &ID{IsLeaf: true, Value: 0}, Right: id.Right}
|
||||
}
|
||||
|
||||
// Join merges two IDs
|
||||
func Join(id1, id2 *ID) *ID {
|
||||
if id1.IsLeaf && id1.Value == 0 {
|
||||
return id2
|
||||
}
|
||||
if id2.IsLeaf && id2.Value == 0 {
|
||||
return id1
|
||||
}
|
||||
if id1.IsLeaf && id2.IsLeaf && id1.Value == 1 && id2.Value == 1 {
|
||||
return &ID{IsLeaf: true, Value: 1}
|
||||
}
|
||||
|
||||
// Normalize to non-leaf
|
||||
left1 := id1.Left
|
||||
right1 := id1.Right
|
||||
left2 := id2.Left
|
||||
right2 := id2.Right
|
||||
|
||||
if id1.IsLeaf {
|
||||
left1 = id1
|
||||
right1 = id1
|
||||
}
|
||||
if id2.IsLeaf {
|
||||
left2 = id2
|
||||
right2 = id2
|
||||
}
|
||||
|
||||
newLeft := Join(left1, left2)
|
||||
newRight := Join(right1, right2)
|
||||
|
||||
return normalize(&ID{IsLeaf: false, Left: newLeft, Right: newRight})
|
||||
}
|
||||
|
||||
func normalize(id *ID) *ID {
|
||||
if !id.IsLeaf {
|
||||
if id.Left.IsLeaf && id.Right.IsLeaf &&
|
||||
id.Left.Value == id.Right.Value {
|
||||
return &ID{IsLeaf: true, Value: id.Left.Value}
|
||||
}
|
||||
}
|
||||
return id
|
||||
}
|
||||
```
|
||||
|
||||
### Stamp Operations
|
||||
|
||||
```go
|
||||
// NewStamp creates initial stamp (0)
|
||||
func NewStamp() *Stamp {
|
||||
return &Stamp{Base: 0}
|
||||
}
|
||||
|
||||
// Event increments the stamp for the given ID
|
||||
func Event(id *ID, stamp *Stamp) *Stamp {
|
||||
if id.IsLeaf {
|
||||
if id.Value == 1 {
|
||||
return &Stamp{Base: stamp.Base + 1}
|
||||
}
|
||||
return stamp // Cannot increment with zero ID
|
||||
}
|
||||
|
||||
// Non-leaf ID: fill where we have ID
|
||||
if id.Left.IsLeaf && id.Left.Value == 1 {
|
||||
// Have left ID, increment left
|
||||
newLeft := Event(&ID{IsLeaf: true, Value: 1}, getLeft(stamp))
|
||||
return normalizeStamp(&Stamp{
|
||||
Base: stamp.Base,
|
||||
Left: newLeft,
|
||||
Right: getRight(stamp),
|
||||
})
|
||||
}
|
||||
if id.Right.IsLeaf && id.Right.Value == 1 {
|
||||
newRight := Event(&ID{IsLeaf: true, Value: 1}, getRight(stamp))
|
||||
return normalizeStamp(&Stamp{
|
||||
Base: stamp.Base,
|
||||
Left: getLeft(stamp),
|
||||
Right: newRight,
|
||||
})
|
||||
}
|
||||
|
||||
// Both non-zero, choose lower side
|
||||
leftMax := maxStamp(getLeft(stamp))
|
||||
rightMax := maxStamp(getRight(stamp))
|
||||
|
||||
if leftMax <= rightMax {
|
||||
return normalizeStamp(&Stamp{
|
||||
Base: stamp.Base,
|
||||
Left: Event(id.Left, getLeft(stamp)),
|
||||
Right: getRight(stamp),
|
||||
})
|
||||
}
|
||||
return normalizeStamp(&Stamp{
|
||||
Base: stamp.Base,
|
||||
Left: getLeft(stamp),
|
||||
Right: Event(id.Right, getRight(stamp)),
|
||||
})
|
||||
}
|
||||
|
||||
func getLeft(s *Stamp) *Stamp {
|
||||
if s.Left == nil {
|
||||
return &Stamp{Base: 0}
|
||||
}
|
||||
return s.Left
|
||||
}
|
||||
|
||||
func getRight(s *Stamp) *Stamp {
|
||||
if s.Right == nil {
|
||||
return &Stamp{Base: 0}
|
||||
}
|
||||
return s.Right
|
||||
}
|
||||
|
||||
func maxStamp(s *Stamp) int {
|
||||
if s.Left == nil && s.Right == nil {
|
||||
return s.Base
|
||||
}
|
||||
left := 0
|
||||
right := 0
|
||||
if s.Left != nil {
|
||||
left = maxStamp(s.Left)
|
||||
}
|
||||
if s.Right != nil {
|
||||
right = maxStamp(s.Right)
|
||||
}
|
||||
max := left
|
||||
if right > max {
|
||||
max = right
|
||||
}
|
||||
return s.Base + max
|
||||
}
|
||||
|
||||
// JoinStamps merges two stamps
|
||||
func JoinStamps(s1, s2 *Stamp) *Stamp {
|
||||
// Take max at each level
|
||||
base := s1.Base
|
||||
if s2.Base > base {
|
||||
base = s2.Base
|
||||
}
|
||||
|
||||
// Adjust for base difference
|
||||
adj1 := s1.Base
|
||||
adj2 := s2.Base
|
||||
|
||||
return normalizeStamp(&Stamp{
|
||||
Base: base,
|
||||
Left: joinStampsRecursive(s1.Left, s2.Left, adj1-base, adj2-base),
|
||||
Right: joinStampsRecursive(s1.Right, s2.Right, adj1-base, adj2-base),
|
||||
})
|
||||
}
|
||||
|
||||
func normalizeStamp(s *Stamp) *Stamp {
|
||||
if s.Left == nil && s.Right == nil {
|
||||
return s
|
||||
}
|
||||
if s.Left != nil && s.Right != nil {
|
||||
if s.Left.Base > 0 && s.Right.Base > 0 {
|
||||
min := s.Left.Base
|
||||
if s.Right.Base < min {
|
||||
min = s.Right.Base
|
||||
}
|
||||
return &Stamp{
|
||||
Base: s.Base + min,
|
||||
Left: &Stamp{Base: s.Left.Base - min, Left: s.Left.Left, Right: s.Left.Right},
|
||||
Right: &Stamp{Base: s.Right.Base - min, Left: s.Right.Left, Right: s.Right.Right},
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
```
|
||||
|
||||
## Hybrid Logical Clock Implementation
|
||||
|
||||
```go
|
||||
type HLC struct {
|
||||
l int64 // logical component (physical time)
|
||||
c int64 // counter
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewHLC() *HLC {
|
||||
return &HLC{l: 0, c: 0}
|
||||
}
|
||||
|
||||
type HLCTimestamp struct {
|
||||
L int64
|
||||
C int64
|
||||
}
|
||||
|
||||
func (hlc *HLC) physicalTime() int64 {
|
||||
return time.Now().UnixNano()
|
||||
}
|
||||
|
||||
// Now returns current HLC timestamp for local/send event
|
||||
func (hlc *HLC) Now() HLCTimestamp {
|
||||
hlc.mu.Lock()
|
||||
defer hlc.mu.Unlock()
|
||||
|
||||
pt := hlc.physicalTime()
|
||||
|
||||
if pt > hlc.l {
|
||||
hlc.l = pt
|
||||
hlc.c = 0
|
||||
} else {
|
||||
hlc.c++
|
||||
}
|
||||
|
||||
return HLCTimestamp{L: hlc.l, C: hlc.c}
|
||||
}
|
||||
|
||||
// Update updates HLC based on received timestamp
|
||||
func (hlc *HLC) Update(received HLCTimestamp) HLCTimestamp {
|
||||
hlc.mu.Lock()
|
||||
defer hlc.mu.Unlock()
|
||||
|
||||
pt := hlc.physicalTime()
|
||||
|
||||
if pt > hlc.l && pt > received.L {
|
||||
hlc.l = pt
|
||||
hlc.c = 0
|
||||
} else if received.L > hlc.l {
|
||||
hlc.l = received.L
|
||||
hlc.c = received.C + 1
|
||||
} else if hlc.l > received.L {
|
||||
hlc.c++
|
||||
} else { // hlc.l == received.L
|
||||
if received.C > hlc.c {
|
||||
hlc.c = received.C + 1
|
||||
} else {
|
||||
hlc.c++
|
||||
}
|
||||
}
|
||||
|
||||
return HLCTimestamp{L: hlc.l, C: hlc.c}
|
||||
}
|
||||
|
||||
// Compare compares two HLC timestamps
|
||||
func (t1 HLCTimestamp) Compare(t2 HLCTimestamp) int {
|
||||
if t1.L < t2.L {
|
||||
return -1
|
||||
}
|
||||
if t1.L > t2.L {
|
||||
return 1
|
||||
}
|
||||
if t1.C < t2.C {
|
||||
return -1
|
||||
}
|
||||
if t1.C > t2.C {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
```
|
||||
|
||||
## Causal Broadcast Implementation
|
||||
|
||||
```go
|
||||
type CausalBroadcast struct {
|
||||
vc *VectorClock
|
||||
pending []PendingMessage
|
||||
deliver func(Message)
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
type PendingMessage struct {
|
||||
Msg Message
|
||||
Timestamp map[string]uint64
|
||||
}
|
||||
|
||||
func NewCausalBroadcast(processID string, processes []string, deliver func(Message)) *CausalBroadcast {
|
||||
return &CausalBroadcast{
|
||||
vc: NewVectorClock(processID, processes),
|
||||
pending: make([]PendingMessage, 0),
|
||||
deliver: deliver,
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast sends a message to all processes
|
||||
func (cb *CausalBroadcast) Broadcast(msg Message) map[string]uint64 {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
|
||||
timestamp := cb.vc.Send()
|
||||
// Actual network broadcast would happen here
|
||||
return timestamp
|
||||
}
|
||||
|
||||
// Receive handles an incoming message
|
||||
func (cb *CausalBroadcast) Receive(msg Message, sender string, timestamp map[string]uint64) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
|
||||
// Add to pending
|
||||
cb.pending = append(cb.pending, PendingMessage{Msg: msg, Timestamp: timestamp})
|
||||
|
||||
// Try to deliver pending messages
|
||||
cb.tryDeliver()
|
||||
}
|
||||
|
||||
func (cb *CausalBroadcast) tryDeliver() {
|
||||
changed := true
|
||||
for changed {
|
||||
changed = false
|
||||
|
||||
for i, pending := range cb.pending {
|
||||
if cb.canDeliver(pending.Timestamp) {
|
||||
// Deliver message
|
||||
cb.vc.Receive(pending.Timestamp)
|
||||
cb.deliver(pending.Msg)
|
||||
|
||||
// Remove from pending
|
||||
cb.pending = append(cb.pending[:i], cb.pending[i+1:]...)
|
||||
changed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CausalBroadcast) canDeliver(msgVC map[string]uint64) bool {
|
||||
currentVC := cb.vc.clocks
|
||||
|
||||
for pid, msgTime := range msgVC {
|
||||
if pid == cb.vc.self {
|
||||
// Must be next expected from sender
|
||||
if msgTime != currentVC[pid]+1 {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
// All other dependencies must be satisfied
|
||||
if msgTime > currentVC[pid] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
```
|
||||
369
.claude/skills/elliptic-curves/SKILL.md
Normal file
369
.claude/skills/elliptic-curves/SKILL.md
Normal file
@@ -0,0 +1,369 @@
|
||||
---
|
||||
name: elliptic-curves
|
||||
description: This skill should be used when working with elliptic curve cryptography, implementing or debugging secp256k1 operations, understanding modular arithmetic and finite fields, or implementing signature schemes like ECDSA and Schnorr. Provides comprehensive knowledge of group theory foundations, curve mathematics, point multiplication algorithms, and cryptographic optimizations.
|
||||
---
|
||||
|
||||
# Elliptic Curve Cryptography
|
||||
|
||||
This skill provides deep knowledge of elliptic curve cryptography (ECC), with particular focus on the secp256k1 curve used in Bitcoin and Nostr, including the mathematical foundations and implementation considerations.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
- Implementing or debugging elliptic curve operations
|
||||
- Working with secp256k1, ECDSA, or Schnorr signatures
|
||||
- Understanding modular arithmetic and finite field operations
|
||||
- Optimizing cryptographic code for performance
|
||||
- Analyzing security properties of curve-based cryptography
|
||||
|
||||
## Mathematical Foundations
|
||||
|
||||
### Groups in Cryptography
|
||||
|
||||
A **group** is a set G with a binary operation (often denoted · or +) satisfying:
|
||||
|
||||
1. **Closure**: For all a, b ∈ G, the result a · b is also in G
|
||||
2. **Associativity**: (a · b) · c = a · (b · c)
|
||||
3. **Identity**: There exists e ∈ G such that e · a = a · e = a
|
||||
4. **Inverse**: For each a ∈ G, there exists a⁻¹ such that a · a⁻¹ = e
|
||||
|
||||
A **cyclic group** is generated by repeatedly applying the operation to a single element (the generator). The **order** of a group is the number of elements.
|
||||
|
||||
**Why groups matter in cryptography**: The discrete logarithm problem—given g and gⁿ, find n—is computationally hard in certain groups, forming the security basis for ECC.
|
||||
|
||||
### Modular Arithmetic
|
||||
|
||||
Modular arithmetic constrains calculations to a finite range [0, p-1] for some modulus p:
|
||||
|
||||
```
|
||||
a ≡ b (mod p) means p divides (a - b)
|
||||
|
||||
Operations:
|
||||
- Addition: (a + b) mod p
|
||||
- Subtraction: (a - b + p) mod p
|
||||
- Multiplication: (a × b) mod p
|
||||
- Inverse: a⁻¹ where (a × a⁻¹) ≡ 1 (mod p)
|
||||
```
|
||||
|
||||
**Computing modular inverse**:
|
||||
- **Fermat's Little Theorem**: If p is prime, a⁻¹ ≡ a^(p-2) (mod p)
|
||||
- **Extended Euclidean Algorithm**: More efficient for general cases
|
||||
- **SafeGCD Algorithm**: Constant-time, used in libsecp256k1
|
||||
|
||||
### Finite Fields (Galois Fields)
|
||||
|
||||
A **finite field** GF(p) or 𝔽ₚ is a field with a finite number of elements where:
|
||||
- p must be prime (or a prime power for extension fields)
|
||||
- All arithmetic operations are defined and produce elements within the field
|
||||
- Every non-zero element has a multiplicative inverse
|
||||
|
||||
For cryptographic curves like secp256k1, the field is 𝔽ₚ where p is a 256-bit prime.
|
||||
|
||||
**Key property**: The non-zero elements of a finite field form a cyclic group under multiplication.
|
||||
|
||||
## Elliptic Curves
|
||||
|
||||
### The Curve Equation
|
||||
|
||||
An elliptic curve over a finite field 𝔽ₚ is defined by the Weierstrass equation:
|
||||
|
||||
```
|
||||
y² = x³ + ax + b (mod p)
|
||||
```
|
||||
|
||||
The curve must satisfy the non-singularity condition: 4a³ + 27b² ≠ 0
|
||||
|
||||
### Points on the Curve
|
||||
|
||||
A point P = (x, y) is on the curve if it satisfies the equation. The set of all points, plus a special "point at infinity" O (the identity element), forms an abelian group.
|
||||
|
||||
### Point Operations
|
||||
|
||||
**Point Addition (P + Q where P ≠ Q)**:
|
||||
```
|
||||
λ = (y₂ - y₁) / (x₂ - x₁) (mod p)
|
||||
x₃ = λ² - x₁ - x₂ (mod p)
|
||||
y₃ = λ(x₁ - x₃) - y₁ (mod p)
|
||||
```
|
||||
|
||||
**Point Doubling (P + P = 2P)**:
|
||||
```
|
||||
λ = (3x₁² + a) / (2y₁) (mod p)
|
||||
x₃ = λ² - 2x₁ (mod p)
|
||||
y₃ = λ(x₁ - x₃) - y₁ (mod p)
|
||||
```
|
||||
|
||||
**Point at Infinity**: Acts as the identity element; P + O = P for all P.
|
||||
|
||||
**Point Negation**: -P = (x, -y) = (x, p - y)
|
||||
|
||||
## The secp256k1 Curve
|
||||
|
||||
### Parameters
|
||||
|
||||
secp256k1 is defined by SECG (Standards for Efficient Cryptography Group):
|
||||
|
||||
```
|
||||
Curve equation: y² = x³ + 7 (a = 0, b = 7)
|
||||
|
||||
Prime modulus p:
|
||||
0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F
|
||||
= 2²⁵⁶ - 2³² - 977
|
||||
|
||||
Group order n:
|
||||
0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141
|
||||
|
||||
Generator point G:
|
||||
Gx = 0x79BE667E F9DCBBAC 55A06295 CE870B07 029BFCDB 2DCE28D9 59F2815B 16F81798
|
||||
Gy = 0x483ADA77 26A3C465 5DA4FBFC 0E1108A8 FD17B448 A6855419 9C47D08F FB10D4B8
|
||||
|
||||
Cofactor h = 1
|
||||
```
|
||||
|
||||
### Why secp256k1?
|
||||
|
||||
1. **Koblitz curve**: a = 0 enables faster computation (no ax term)
|
||||
2. **Special prime**: p = 2²⁵⁶ - 2³² - 977 allows efficient modular reduction
|
||||
3. **Deterministic construction**: Not randomly generated, reducing backdoor concerns
|
||||
4. **~30% faster** than random curves when fully optimized
|
||||
|
||||
### Efficient Modular Reduction
|
||||
|
||||
The special form of p enables fast reduction without general division:
|
||||
|
||||
```
|
||||
For p = 2²⁵⁶ - 2³² - 977:
|
||||
To reduce a 512-bit number c = c_high × 2²⁵⁶ + c_low:
|
||||
c ≡ c_low + c_high × 2³² + c_high × 977 (mod p)
|
||||
```
|
||||
|
||||
## Point Multiplication Algorithms
|
||||
|
||||
Scalar multiplication kP (computing P + P + ... + P, k times) is the core operation.
|
||||
|
||||
### Double-and-Add (Binary Method)
|
||||
|
||||
```
|
||||
Input: k (scalar), P (point)
|
||||
Output: kP
|
||||
|
||||
R = O (point at infinity)
|
||||
for i from bit_length(k)-1 down to 0:
|
||||
R = 2R # Point doubling
|
||||
if bit i of k is 1:
|
||||
R = R + P # Point addition
|
||||
return R
|
||||
```
|
||||
|
||||
**Complexity**: O(log k) point operations
|
||||
**Vulnerability**: Timing side-channels (different branches for 0/1 bits)
|
||||
|
||||
### Montgomery Ladder
|
||||
|
||||
Constant-time algorithm that performs the same operations regardless of bit values:
|
||||
|
||||
```
|
||||
Input: k (scalar), P (point)
|
||||
Output: kP
|
||||
|
||||
R0 = O
|
||||
R1 = P
|
||||
for i from bit_length(k)-1 down to 0:
|
||||
if bit i of k is 0:
|
||||
R1 = R0 + R1
|
||||
R0 = 2R0
|
||||
else:
|
||||
R0 = R0 + R1
|
||||
R1 = 2R1
|
||||
return R0
|
||||
```
|
||||
|
||||
**Advantage**: Resistant to simple power analysis and timing attacks.
|
||||
|
||||
### Window Methods (w-NAF)
|
||||
|
||||
Precompute small multiples of P, then process w bits at a time:
|
||||
|
||||
```
|
||||
w-NAF representation reduces additions by ~1/3 compared to binary
|
||||
Precomputation table: [P, 3P, 5P, 7P, ...] for w=4
|
||||
```
|
||||
|
||||
### Endomorphism Optimization (GLV Method)
|
||||
|
||||
secp256k1 has an efficiently computable endomorphism φ where:
|
||||
```
|
||||
φ(x, y) = (βx, y) where β³ ≡ 1 (mod p)
|
||||
φ(P) = λP where λ³ ≡ 1 (mod n)
|
||||
```
|
||||
|
||||
This allows splitting scalar k into k₁ + k₂λ with smaller k₁, k₂, reducing operations by ~33-50%.
|
||||
|
||||
### Multi-Scalar Multiplication (Strauss-Shamir)
|
||||
|
||||
For computing k₁P₁ + k₂P₂ (common in signature verification):
|
||||
|
||||
```
|
||||
Process both scalars simultaneously, combining operations
|
||||
Reduces work compared to separate multiplications
|
||||
```
|
||||
|
||||
## Coordinate Systems
|
||||
|
||||
### Affine Coordinates
|
||||
|
||||
Standard (x, y) representation. Requires modular inversion for each operation.
|
||||
|
||||
### Projective Coordinates
|
||||
|
||||
Represent (X:Y:Z) where x = X/Z, y = Y/Z:
|
||||
- Avoids inversions during intermediate computations
|
||||
- Only one inversion at the end to convert back to affine
|
||||
|
||||
### Jacobian Coordinates
|
||||
|
||||
Represent (X:Y:Z) where x = X/Z², y = Y/Z³:
|
||||
- Fastest for point doubling
|
||||
- Used extensively in libsecp256k1
|
||||
|
||||
### López-Dahab Coordinates
|
||||
|
||||
For curves over GF(2ⁿ), optimized for binary field arithmetic.
|
||||
|
||||
## Signature Schemes
|
||||
|
||||
### ECDSA (Elliptic Curve Digital Signature Algorithm)
|
||||
|
||||
**Key Generation**:
|
||||
```
|
||||
Private key: d (random integer in [1, n-1])
|
||||
Public key: Q = dG
|
||||
```
|
||||
|
||||
**Signing message m**:
|
||||
```
|
||||
1. Hash: e = H(m) truncated to curve order bit length
|
||||
2. Random: k ∈ [1, n-1]
|
||||
3. Compute: (x, y) = kG
|
||||
4. Calculate: r = x mod n (if r = 0, restart with new k)
|
||||
5. Calculate: s = k⁻¹(e + rd) mod n (if s = 0, restart)
|
||||
6. Signature: (r, s)
|
||||
```
|
||||
|
||||
**Verification of signature (r, s) on message m**:
|
||||
```
|
||||
1. Check: r, s ∈ [1, n-1]
|
||||
2. Hash: e = H(m)
|
||||
3. Compute: w = s⁻¹ mod n
|
||||
4. Compute: u₁ = ew mod n, u₂ = rw mod n
|
||||
5. Compute: (x, y) = u₁G + u₂Q
|
||||
6. Valid if: r ≡ x (mod n)
|
||||
```
|
||||
|
||||
**Security considerations**:
|
||||
- k MUST be unique per signature (reuse leaks private key)
|
||||
- Use RFC 6979 for deterministic k derivation
|
||||
|
||||
### Schnorr Signatures (BIP-340)
|
||||
|
||||
Simpler, more efficient, with provable security.
|
||||
|
||||
**Signing message m**:
|
||||
```
|
||||
1. Random: k ∈ [1, n-1]
|
||||
2. Compute: R = kG
|
||||
3. Challenge: e = H(R || Q || m)
|
||||
4. Response: s = k + ed mod n
|
||||
5. Signature: (R, s) or (r_x, s) where r_x is x-coordinate of R
|
||||
```
|
||||
|
||||
**Verification**:
|
||||
```
|
||||
1. Compute: e = H(R || Q || m)
|
||||
2. Check: sG = R + eQ
|
||||
```
|
||||
|
||||
**Advantages over ECDSA**:
|
||||
- Linear: enables signature aggregation (MuSig)
|
||||
- Simpler verification (no modular inverse)
|
||||
- Batch verification support
|
||||
- Provably secure in Random Oracle Model
|
||||
|
||||
## Implementation Considerations
|
||||
|
||||
### Constant-Time Operations
|
||||
|
||||
To prevent timing attacks:
|
||||
- Avoid branches dependent on secret data
|
||||
- Use constant-time comparison functions
|
||||
- Mask operations to hide data-dependent timing
|
||||
|
||||
```go
|
||||
// BAD: Timing leak
|
||||
if secretBit == 1 {
|
||||
doOperation()
|
||||
}
|
||||
|
||||
// GOOD: Constant-time conditional
|
||||
result = conditionalSelect(secretBit, value1, value0)
|
||||
```
|
||||
|
||||
### Memory Safety
|
||||
|
||||
- Zeroize sensitive data after use
|
||||
- Avoid leaving secrets in registers or cache
|
||||
- Use secure memory allocation when available
|
||||
|
||||
### Side-Channel Protections
|
||||
|
||||
- **Timing attacks**: Use constant-time algorithms
|
||||
- **Power analysis**: Montgomery ladder, point blinding
|
||||
- **Cache attacks**: Avoid table lookups indexed by secrets
|
||||
|
||||
### Random Number Generation
|
||||
|
||||
- Use cryptographically secure RNG for k in ECDSA
|
||||
- Consider deterministic k (RFC 6979) for reproducibility
|
||||
- Validate output is in valid range [1, n-1]
|
||||
|
||||
## libsecp256k1 Optimizations
|
||||
|
||||
The Bitcoin Core library includes:
|
||||
|
||||
1. **Field arithmetic**: 5×52-bit limbs for 64-bit platforms
|
||||
2. **Scalar arithmetic**: 4×64-bit representation
|
||||
3. **Endomorphism**: GLV decomposition enabled by default
|
||||
4. **Batch inversion**: Amortizes expensive inversions
|
||||
5. **SafeGCD**: Constant-time modular inverse
|
||||
6. **Precomputed tables**: For generator point multiplications
|
||||
|
||||
## Security Properties
|
||||
|
||||
### Discrete Logarithm Problem (DLP)
|
||||
|
||||
Given P and Q = kP, finding k is computationally infeasible.
|
||||
|
||||
**Best known attacks**:
|
||||
- Generic: Baby-step Giant-step, Pollard's rho: O(√n) operations
|
||||
- For secp256k1: ~2¹²⁸ operations (128-bit security)
|
||||
|
||||
### Curve Security Criteria
|
||||
|
||||
- Large prime order subgroup
|
||||
- Cofactor 1 (no small subgroup attacks)
|
||||
- Resistant to MOV attack (embedding degree)
|
||||
- Not anomalous (n ≠ p)
|
||||
|
||||
## Common Pitfalls
|
||||
|
||||
1. **k reuse in ECDSA**: Immediately leaks private key
|
||||
2. **Weak random k**: Partially leaks key over multiple signatures
|
||||
3. **Invalid curve points**: Validate points are on curve
|
||||
4. **Small subgroup attacks**: Check point order (cofactor = 1 helps)
|
||||
5. **Timing leaks**: Non-constant-time scalar multiplication
|
||||
|
||||
## References
|
||||
|
||||
For detailed implementations, see:
|
||||
- `references/secp256k1-parameters.md` - Full curve parameters
|
||||
- `references/algorithms.md` - Detailed algorithm pseudocode
|
||||
- `references/security.md` - Security analysis and attack vectors
|
||||
513
.claude/skills/elliptic-curves/references/algorithms.md
Normal file
513
.claude/skills/elliptic-curves/references/algorithms.md
Normal file
@@ -0,0 +1,513 @@
|
||||
# Elliptic Curve Algorithms
|
||||
|
||||
Detailed pseudocode for core elliptic curve operations.
|
||||
|
||||
## Field Arithmetic
|
||||
|
||||
### Modular Addition
|
||||
|
||||
```
|
||||
function mod_add(a, b, p):
|
||||
result = a + b
|
||||
if result >= p:
|
||||
result = result - p
|
||||
return result
|
||||
```
|
||||
|
||||
### Modular Subtraction
|
||||
|
||||
```
|
||||
function mod_sub(a, b, p):
|
||||
if a >= b:
|
||||
return a - b
|
||||
else:
|
||||
return p - b + a
|
||||
```
|
||||
|
||||
### Modular Multiplication
|
||||
|
||||
For general case:
|
||||
```
|
||||
function mod_mul(a, b, p):
|
||||
return (a * b) mod p
|
||||
```
|
||||
|
||||
For secp256k1 optimized (Barrett reduction):
|
||||
```
|
||||
function mod_mul_secp256k1(a, b):
|
||||
# Compute full 512-bit product
|
||||
product = a * b
|
||||
|
||||
# Split into high and low 256-bit parts
|
||||
low = product & ((1 << 256) - 1)
|
||||
high = product >> 256
|
||||
|
||||
# Reduce: result ≡ low + high * (2³² + 977) (mod p)
|
||||
result = low + high * (1 << 32) + high * 977
|
||||
|
||||
# May need additional reduction
|
||||
while result >= p:
|
||||
result = result - p
|
||||
|
||||
return result
|
||||
```
|
||||
|
||||
### Modular Inverse
|
||||
|
||||
**Extended Euclidean Algorithm**:
|
||||
```
|
||||
function mod_inverse(a, p):
|
||||
if a == 0:
|
||||
error "No inverse exists for 0"
|
||||
|
||||
old_r, r = p, a
|
||||
old_s, s = 0, 1
|
||||
|
||||
while r != 0:
|
||||
quotient = old_r / r
|
||||
old_r, r = r, old_r - quotient * r
|
||||
old_s, s = s, old_s - quotient * s
|
||||
|
||||
if old_r != 1:
|
||||
error "No inverse exists"
|
||||
|
||||
if old_s < 0:
|
||||
old_s = old_s + p
|
||||
|
||||
return old_s
|
||||
```
|
||||
|
||||
**Fermat's Little Theorem** (for prime p):
|
||||
```
|
||||
function mod_inverse_fermat(a, p):
|
||||
return mod_exp(a, p - 2, p)
|
||||
```
|
||||
|
||||
### Modular Exponentiation (Square-and-Multiply)
|
||||
|
||||
```
|
||||
function mod_exp(base, exp, p):
|
||||
result = 1
|
||||
base = base mod p
|
||||
|
||||
while exp > 0:
|
||||
if exp & 1: # exp is odd
|
||||
result = (result * base) mod p
|
||||
exp = exp >> 1
|
||||
base = (base * base) mod p
|
||||
|
||||
return result
|
||||
```
|
||||
|
||||
### Modular Square Root (Tonelli-Shanks)
|
||||
|
||||
For secp256k1 where p ≡ 3 (mod 4):
|
||||
```
|
||||
function mod_sqrt(a, p):
|
||||
# For p ≡ 3 (mod 4), sqrt(a) = a^((p+1)/4)
|
||||
return mod_exp(a, (p + 1) / 4, p)
|
||||
```
|
||||
|
||||
## Point Operations
|
||||
|
||||
### Point Validation
|
||||
|
||||
```
|
||||
function is_on_curve(P, a, b, p):
|
||||
if P is infinity:
|
||||
return true
|
||||
|
||||
x, y = P
|
||||
left = (y * y) mod p
|
||||
right = (x * x * x + a * x + b) mod p
|
||||
|
||||
return left == right
|
||||
```
|
||||
|
||||
### Point Addition (Affine Coordinates)
|
||||
|
||||
```
|
||||
function point_add(P, Q, a, p):
|
||||
if P is infinity:
|
||||
return Q
|
||||
if Q is infinity:
|
||||
return P
|
||||
|
||||
x1, y1 = P
|
||||
x2, y2 = Q
|
||||
|
||||
if x1 == x2:
|
||||
if y1 == mod_neg(y2, p): # P = -Q
|
||||
return infinity
|
||||
else: # P == Q
|
||||
return point_double(P, a, p)
|
||||
|
||||
# λ = (y2 - y1) / (x2 - x1)
|
||||
numerator = mod_sub(y2, y1, p)
|
||||
denominator = mod_sub(x2, x1, p)
|
||||
λ = mod_mul(numerator, mod_inverse(denominator, p), p)
|
||||
|
||||
# x3 = λ² - x1 - x2
|
||||
x3 = mod_sub(mod_sub(mod_mul(λ, λ, p), x1, p), x2, p)
|
||||
|
||||
# y3 = λ(x1 - x3) - y1
|
||||
y3 = mod_sub(mod_mul(λ, mod_sub(x1, x3, p), p), y1, p)
|
||||
|
||||
return (x3, y3)
|
||||
```
|
||||
|
||||
### Point Doubling (Affine Coordinates)
|
||||
|
||||
```
|
||||
function point_double(P, a, p):
|
||||
if P is infinity:
|
||||
return infinity
|
||||
|
||||
x, y = P
|
||||
|
||||
if y == 0:
|
||||
return infinity
|
||||
|
||||
# λ = (3x² + a) / (2y)
|
||||
numerator = mod_add(mod_mul(3, mod_mul(x, x, p), p), a, p)
|
||||
denominator = mod_mul(2, y, p)
|
||||
λ = mod_mul(numerator, mod_inverse(denominator, p), p)
|
||||
|
||||
# x3 = λ² - 2x
|
||||
x3 = mod_sub(mod_mul(λ, λ, p), mod_mul(2, x, p), p)
|
||||
|
||||
# y3 = λ(x - x3) - y
|
||||
y3 = mod_sub(mod_mul(λ, mod_sub(x, x3, p), p), y, p)
|
||||
|
||||
return (x3, y3)
|
||||
```
|
||||
|
||||
### Point Negation
|
||||
|
||||
```
|
||||
function point_negate(P, p):
|
||||
if P is infinity:
|
||||
return infinity
|
||||
|
||||
x, y = P
|
||||
return (x, p - y)
|
||||
```
|
||||
|
||||
## Scalar Multiplication
|
||||
|
||||
### Double-and-Add (Left-to-Right)
|
||||
|
||||
```
|
||||
function scalar_mult_double_add(k, P, a, p):
|
||||
if k == 0 or P is infinity:
|
||||
return infinity
|
||||
|
||||
if k < 0:
|
||||
k = -k
|
||||
P = point_negate(P, p)
|
||||
|
||||
R = infinity
|
||||
bits = binary_representation(k) # MSB first
|
||||
|
||||
for bit in bits:
|
||||
R = point_double(R, a, p)
|
||||
if bit == 1:
|
||||
R = point_add(R, P, a, p)
|
||||
|
||||
return R
|
||||
```
|
||||
|
||||
### Montgomery Ladder (Constant-Time)
|
||||
|
||||
```
|
||||
function scalar_mult_montgomery(k, P, a, p):
|
||||
R0 = infinity
|
||||
R1 = P
|
||||
|
||||
bits = binary_representation(k) # MSB first
|
||||
|
||||
for bit in bits:
|
||||
if bit == 0:
|
||||
R1 = point_add(R0, R1, a, p)
|
||||
R0 = point_double(R0, a, p)
|
||||
else:
|
||||
R0 = point_add(R0, R1, a, p)
|
||||
R1 = point_double(R1, a, p)
|
||||
|
||||
return R0
|
||||
```
|
||||
|
||||
### w-NAF Scalar Multiplication
|
||||
|
||||
```
|
||||
function compute_wNAF(k, w):
|
||||
# Convert scalar to width-w Non-Adjacent Form
|
||||
naf = []
|
||||
|
||||
while k > 0:
|
||||
if k & 1: # k is odd
|
||||
# Get w-bit window
|
||||
digit = k mod (1 << w)
|
||||
if digit >= (1 << (w-1)):
|
||||
digit = digit - (1 << w)
|
||||
naf.append(digit)
|
||||
k = k - digit
|
||||
else:
|
||||
naf.append(0)
|
||||
k = k >> 1
|
||||
|
||||
return naf
|
||||
|
||||
function scalar_mult_wNAF(k, P, w, a, p):
|
||||
# Precompute odd multiples: [P, 3P, 5P, ..., (2^(w-1)-1)P]
|
||||
precomp = [P]
|
||||
P2 = point_double(P, a, p)
|
||||
for i in range(1, 1 << (w-1)):
|
||||
precomp.append(point_add(precomp[-1], P2, a, p))
|
||||
|
||||
# Convert k to w-NAF
|
||||
naf = compute_wNAF(k, w)
|
||||
|
||||
# Compute scalar multiplication
|
||||
R = infinity
|
||||
for i in range(len(naf) - 1, -1, -1):
|
||||
R = point_double(R, a, p)
|
||||
digit = naf[i]
|
||||
if digit > 0:
|
||||
R = point_add(R, precomp[(digit - 1) / 2], a, p)
|
||||
elif digit < 0:
|
||||
R = point_add(R, point_negate(precomp[(-digit - 1) / 2], p), a, p)
|
||||
|
||||
return R
|
||||
```
|
||||
|
||||
### Shamir's Trick (Multi-Scalar)
|
||||
|
||||
For computing k₁P + k₂Q efficiently:
|
||||
|
||||
```
|
||||
function multi_scalar_mult(k1, P, k2, Q, a, p):
|
||||
# Precompute P + Q
|
||||
PQ = point_add(P, Q, a, p)
|
||||
|
||||
# Get binary representations (same length, padded)
|
||||
bits1 = binary_representation(k1)
|
||||
bits2 = binary_representation(k2)
|
||||
max_len = max(len(bits1), len(bits2))
|
||||
bits1 = pad_left(bits1, max_len)
|
||||
bits2 = pad_left(bits2, max_len)
|
||||
|
||||
R = infinity
|
||||
|
||||
for i in range(max_len):
|
||||
R = point_double(R, a, p)
|
||||
|
||||
b1, b2 = bits1[i], bits2[i]
|
||||
|
||||
if b1 == 1 and b2 == 1:
|
||||
R = point_add(R, PQ, a, p)
|
||||
elif b1 == 1:
|
||||
R = point_add(R, P, a, p)
|
||||
elif b2 == 1:
|
||||
R = point_add(R, Q, a, p)
|
||||
|
||||
return R
|
||||
```
|
||||
|
||||
## Jacobian Coordinates
|
||||
|
||||
More efficient for repeated operations.
|
||||
|
||||
### Conversion
|
||||
|
||||
```
|
||||
# Affine to Jacobian
|
||||
function affine_to_jacobian(P):
|
||||
if P is infinity:
|
||||
return (1, 1, 0) # Jacobian infinity
|
||||
x, y = P
|
||||
return (x, y, 1)
|
||||
|
||||
# Jacobian to Affine
|
||||
function jacobian_to_affine(P, p):
|
||||
X, Y, Z = P
|
||||
if Z == 0:
|
||||
return infinity
|
||||
|
||||
Z_inv = mod_inverse(Z, p)
|
||||
Z_inv2 = mod_mul(Z_inv, Z_inv, p)
|
||||
Z_inv3 = mod_mul(Z_inv2, Z_inv, p)
|
||||
|
||||
x = mod_mul(X, Z_inv2, p)
|
||||
y = mod_mul(Y, Z_inv3, p)
|
||||
|
||||
return (x, y)
|
||||
```
|
||||
|
||||
### Point Doubling (Jacobian)
|
||||
|
||||
For curve y² = x³ + 7 (a = 0):
|
||||
|
||||
```
|
||||
function jacobian_double(P, p):
|
||||
X, Y, Z = P
|
||||
|
||||
if Y == 0:
|
||||
return (1, 1, 0) # infinity
|
||||
|
||||
# For a = 0: M = 3*X²
|
||||
S = mod_mul(4, mod_mul(X, mod_mul(Y, Y, p), p), p)
|
||||
M = mod_mul(3, mod_mul(X, X, p), p)
|
||||
|
||||
X3 = mod_sub(mod_mul(M, M, p), mod_mul(2, S, p), p)
|
||||
Y3 = mod_sub(mod_mul(M, mod_sub(S, X3, p), p),
|
||||
mod_mul(8, mod_mul(Y, Y, mod_mul(Y, Y, p), p), p), p)
|
||||
Z3 = mod_mul(2, mod_mul(Y, Z, p), p)
|
||||
|
||||
return (X3, Y3, Z3)
|
||||
```
|
||||
|
||||
### Point Addition (Jacobian + Affine)
|
||||
|
||||
Mixed addition is faster when one point is in affine:
|
||||
|
||||
```
|
||||
function jacobian_add_affine(P, Q, p):
|
||||
# P in Jacobian (X1, Y1, Z1), Q in affine (x2, y2)
|
||||
X1, Y1, Z1 = P
|
||||
x2, y2 = Q
|
||||
|
||||
if Z1 == 0:
|
||||
return affine_to_jacobian(Q)
|
||||
|
||||
Z1Z1 = mod_mul(Z1, Z1, p)
|
||||
U2 = mod_mul(x2, Z1Z1, p)
|
||||
S2 = mod_mul(y2, mod_mul(Z1, Z1Z1, p), p)
|
||||
|
||||
H = mod_sub(U2, X1, p)
|
||||
HH = mod_mul(H, H, p)
|
||||
I = mod_mul(4, HH, p)
|
||||
J = mod_mul(H, I, p)
|
||||
r = mod_mul(2, mod_sub(S2, Y1, p), p)
|
||||
V = mod_mul(X1, I, p)
|
||||
|
||||
X3 = mod_sub(mod_sub(mod_mul(r, r, p), J, p), mod_mul(2, V, p), p)
|
||||
Y3 = mod_sub(mod_mul(r, mod_sub(V, X3, p), p), mod_mul(2, mod_mul(Y1, J, p), p), p)
|
||||
Z3 = mod_mul(mod_sub(mod_mul(mod_add(Z1, H, p), mod_add(Z1, H, p), p),
|
||||
mod_add(Z1Z1, HH, p), p), 1, p)
|
||||
|
||||
return (X3, Y3, Z3)
|
||||
```
|
||||
|
||||
## GLV Endomorphism (secp256k1)
|
||||
|
||||
### Scalar Decomposition
|
||||
|
||||
```
|
||||
# Constants for secp256k1
|
||||
LAMBDA = 0x5363AD4CC05C30E0A5261C028812645A122E22EA20816678DF02967C1B23BD72
|
||||
BETA = 0x7AE96A2B657C07106E64479EAC3434E99CF0497512F58995C1396C28719501EE
|
||||
|
||||
# Decomposition coefficients
|
||||
A1 = 0x3086D221A7D46BCDE86C90E49284EB15
|
||||
B1 = 0x114CA50F7A8E2F3F657C1108D9D44CFD8
|
||||
A2 = 0xE4437ED6010E88286F547FA90ABFE4C3
|
||||
B2 = A1
|
||||
|
||||
function glv_decompose(k, n):
|
||||
# Compute c1 = round(b2 * k / n)
|
||||
# Compute c2 = round(-b1 * k / n)
|
||||
c1 = (B2 * k + n // 2) // n
|
||||
c2 = (-B1 * k + n // 2) // n
|
||||
|
||||
# k1 = k - c1*A1 - c2*A2
|
||||
# k2 = -c1*B1 - c2*B2
|
||||
k1 = k - c1 * A1 - c2 * A2
|
||||
k2 = -c1 * B1 - c2 * B2
|
||||
|
||||
return (k1, k2)
|
||||
|
||||
function glv_scalar_mult(k, P, p, n):
|
||||
k1, k2 = glv_decompose(k, n)
|
||||
|
||||
# Compute endomorphism: φ(P) = (β*x, y)
|
||||
x, y = P
|
||||
phi_P = (mod_mul(BETA, x, p), y)
|
||||
|
||||
# Use Shamir's trick: k1*P + k2*φ(P)
|
||||
return multi_scalar_mult(k1, P, k2, phi_P, 0, p)
|
||||
```
|
||||
|
||||
## Batch Inversion
|
||||
|
||||
Amortize expensive inversions over multiple points:
|
||||
|
||||
```
|
||||
function batch_invert(values, p):
|
||||
n = len(values)
|
||||
if n == 0:
|
||||
return []
|
||||
|
||||
# Compute cumulative products
|
||||
products = [values[0]]
|
||||
for i in range(1, n):
|
||||
products.append(mod_mul(products[-1], values[i], p))
|
||||
|
||||
# Invert the final product
|
||||
inv = mod_inverse(products[-1], p)
|
||||
|
||||
# Compute individual inverses
|
||||
inverses = [0] * n
|
||||
for i in range(n - 1, 0, -1):
|
||||
inverses[i] = mod_mul(inv, products[i - 1], p)
|
||||
inv = mod_mul(inv, values[i], p)
|
||||
inverses[0] = inv
|
||||
|
||||
return inverses
|
||||
```
|
||||
|
||||
## Key Generation
|
||||
|
||||
```
|
||||
function generate_keypair(G, n, p):
|
||||
# Generate random private key
|
||||
d = random_integer(1, n - 1)
|
||||
|
||||
# Compute public key
|
||||
Q = scalar_mult(d, G)
|
||||
|
||||
return (d, Q)
|
||||
```
|
||||
|
||||
## Point Compression/Decompression
|
||||
|
||||
```
|
||||
function compress_point(P, p):
|
||||
if P is infinity:
|
||||
return bytes([0x00])
|
||||
|
||||
x, y = P
|
||||
prefix = 0x02 if (y % 2 == 0) else 0x03
|
||||
return bytes([prefix]) + x.to_bytes(32, 'big')
|
||||
|
||||
function decompress_point(compressed, a, b, p):
|
||||
prefix = compressed[0]
|
||||
|
||||
if prefix == 0x00:
|
||||
return infinity
|
||||
|
||||
x = int.from_bytes(compressed[1:], 'big')
|
||||
|
||||
# Compute y² = x³ + ax + b
|
||||
y_squared = mod_add(mod_add(mod_mul(x, mod_mul(x, x, p), p),
|
||||
mod_mul(a, x, p), p), b, p)
|
||||
|
||||
# Compute y = sqrt(y²)
|
||||
y = mod_sqrt(y_squared, p)
|
||||
|
||||
# Select correct y based on prefix
|
||||
if (prefix == 0x02) != (y % 2 == 0):
|
||||
y = p - y
|
||||
|
||||
return (x, y)
|
||||
```
|
||||
@@ -0,0 +1,194 @@
|
||||
# secp256k1 Complete Parameters
|
||||
|
||||
## Curve Definition
|
||||
|
||||
**Name**: secp256k1 (Standards for Efficient Cryptography, prime field, 256-bit, Koblitz curve #1)
|
||||
|
||||
**Equation**: y² = x³ + 7 (mod p)
|
||||
|
||||
This is the short Weierstrass form with coefficients a = 0, b = 7.
|
||||
|
||||
## Field Parameters
|
||||
|
||||
### Prime Modulus p
|
||||
|
||||
```
|
||||
Decimal:
|
||||
115792089237316195423570985008687907853269984665640564039457584007908834671663
|
||||
|
||||
Hexadecimal:
|
||||
0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F
|
||||
|
||||
Binary representation:
|
||||
2²⁵⁶ - 2³² - 2⁹ - 2⁸ - 2⁷ - 2⁶ - 2⁴ - 1
|
||||
= 2²⁵⁶ - 2³² - 977
|
||||
```
|
||||
|
||||
**Special form benefits**:
|
||||
- Efficient modular reduction using: c mod p = c_low + c_high × (2³² + 977)
|
||||
- Near-Mersenne prime enables fast arithmetic
|
||||
|
||||
### Group Order n
|
||||
|
||||
```
|
||||
Decimal:
|
||||
115792089237316195423570985008687907852837564279074904382605163141518161494337
|
||||
|
||||
Hexadecimal:
|
||||
0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141
|
||||
```
|
||||
|
||||
The number of points on the curve, including the point at infinity.
|
||||
|
||||
### Cofactor h
|
||||
|
||||
```
|
||||
h = 1
|
||||
```
|
||||
|
||||
Cofactor 1 means the group order n equals the curve order, simplifying security analysis and eliminating small subgroup attacks.
|
||||
|
||||
## Generator Point G
|
||||
|
||||
### Compressed Form
|
||||
|
||||
```
|
||||
02 79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798
|
||||
```
|
||||
|
||||
The 02 prefix indicates the y-coordinate is even.
|
||||
|
||||
### Uncompressed Form
|
||||
|
||||
```
|
||||
04 79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798
|
||||
483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8
|
||||
```
|
||||
|
||||
### Individual Coordinates
|
||||
|
||||
**Gx**:
|
||||
```
|
||||
Decimal:
|
||||
55066263022277343669578718895168534326250603453777594175500187360389116729240
|
||||
|
||||
Hexadecimal:
|
||||
0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798
|
||||
```
|
||||
|
||||
**Gy**:
|
||||
```
|
||||
Decimal:
|
||||
32670510020758816978083085130507043184471273380659243275938904335757337482424
|
||||
|
||||
Hexadecimal:
|
||||
0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8
|
||||
```
|
||||
|
||||
## Endomorphism Parameters
|
||||
|
||||
secp256k1 has an efficiently computable endomorphism φ: (x, y) → (βx, y).
|
||||
|
||||
### β (Beta)
|
||||
|
||||
```
|
||||
Hexadecimal:
|
||||
0x7AE96A2B657C07106E64479EAC3434E99CF0497512F58995C1396C28719501EE
|
||||
|
||||
Property: β³ ≡ 1 (mod p)
|
||||
```
|
||||
|
||||
### λ (Lambda)
|
||||
|
||||
```
|
||||
Hexadecimal:
|
||||
0x5363AD4CC05C30E0A5261C028812645A122E22EA20816678DF02967C1B23BD72
|
||||
|
||||
Property: λ³ ≡ 1 (mod n)
|
||||
Relationship: φ(P) = λP for all points P
|
||||
```
|
||||
|
||||
### GLV Decomposition Constants
|
||||
|
||||
For splitting scalar k into k₁ + k₂λ:
|
||||
|
||||
```
|
||||
a₁ = 0x3086D221A7D46BCDE86C90E49284EB15
|
||||
b₁ = -0xE4437ED6010E88286F547FA90ABFE4C3
|
||||
a₂ = 0x114CA50F7A8E2F3F657C1108D9D44CFD8
|
||||
b₂ = a₁
|
||||
```
|
||||
|
||||
## Derived Constants
|
||||
|
||||
### Field Characteristics
|
||||
|
||||
```
|
||||
(p + 1) / 4 = 0x3FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFBFFFFF0C
|
||||
Used for computing modular square roots via Tonelli-Shanks shortcut
|
||||
```
|
||||
|
||||
### Order Characteristics
|
||||
|
||||
```
|
||||
(n - 1) / 2 = 0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5D576E7357A4501DDFE92F46681B20A0
|
||||
Used in low-S normalization for ECDSA signatures
|
||||
```
|
||||
|
||||
## Validation Formulas
|
||||
|
||||
### Point on Curve Check
|
||||
|
||||
For point (x, y), verify:
|
||||
```
|
||||
y² ≡ x³ + 7 (mod p)
|
||||
```
|
||||
|
||||
### Generator Verification
|
||||
|
||||
Verify G is on curve:
|
||||
```
|
||||
Gy² mod p = 0x9C47D08FFB10D4B8 ... (truncated for display)
|
||||
Gx³ + 7 mod p = same value
|
||||
```
|
||||
|
||||
### Order Verification
|
||||
|
||||
Verify nG = O (point at infinity):
|
||||
```
|
||||
Computing n × G should yield the identity element
|
||||
```
|
||||
|
||||
## Bit Lengths
|
||||
|
||||
| Parameter | Bits | Bytes |
|
||||
|-----------|------|-------|
|
||||
| p (prime) | 256 | 32 |
|
||||
| n (order) | 256 | 32 |
|
||||
| Private key | 256 | 32 |
|
||||
| Public key (compressed) | 257 | 33 |
|
||||
| Public key (uncompressed) | 513 | 65 |
|
||||
| ECDSA signature | 512 | 64 |
|
||||
| Schnorr signature | 512 | 64 |
|
||||
|
||||
## Security Level
|
||||
|
||||
- **Equivalent symmetric key strength**: 128 bits
|
||||
- **Best known attack complexity**: ~2¹²⁸ operations (Pollard's rho)
|
||||
- **Safe until**: Quantum computers with ~1500+ logical qubits
|
||||
|
||||
## ASN.1 OID
|
||||
|
||||
```
|
||||
1.3.132.0.10
|
||||
iso(1) identified-organization(3) certicom(132) curve(0) secp256k1(10)
|
||||
```
|
||||
|
||||
## Comparison with Other Curves
|
||||
|
||||
| Curve | Field Size | Security | Speed | Use Case |
|
||||
|-------|------------|----------|-------|----------|
|
||||
| secp256k1 | 256-bit | 128-bit | Fast (Koblitz) | Bitcoin, Nostr |
|
||||
| secp256r1 (P-256) | 256-bit | 128-bit | Moderate | TLS, general |
|
||||
| Curve25519 | 255-bit | ~128-bit | Very fast | Modern crypto |
|
||||
| secp384r1 (P-384) | 384-bit | 192-bit | Slower | High security |
|
||||
291
.claude/skills/elliptic-curves/references/security.md
Normal file
291
.claude/skills/elliptic-curves/references/security.md
Normal file
@@ -0,0 +1,291 @@
|
||||
# Elliptic Curve Security Analysis
|
||||
|
||||
Security properties, attack vectors, and mitigations for elliptic curve cryptography.
|
||||
|
||||
## The Discrete Logarithm Problem (ECDLP)
|
||||
|
||||
### Definition
|
||||
|
||||
Given points P and Q = kP on an elliptic curve, find the scalar k.
|
||||
|
||||
**Security assumption**: For properly chosen curves, this problem is computationally infeasible.
|
||||
|
||||
### Best Known Attacks
|
||||
|
||||
#### Generic Attacks (Work on Any Group)
|
||||
|
||||
| Attack | Complexity | Notes |
|
||||
|--------|------------|-------|
|
||||
| Baby-step Giant-step | O(√n) space and time | Requires √n storage |
|
||||
| Pollard's rho | O(√n) time, O(1) space | Practical for large groups |
|
||||
| Pollard's lambda | O(√n) | When k is in known range |
|
||||
| Pohlig-Hellman | O(√p) where p is largest prime factor | Exploits factorization of n |
|
||||
|
||||
For secp256k1 (n ≈ 2²⁵⁶):
|
||||
- Generic attack complexity: ~2¹²⁸ operations
|
||||
- Equivalent to 128-bit symmetric security
|
||||
|
||||
#### Curve-Specific Attacks
|
||||
|
||||
| Attack | Applicable When | Mitigation |
|
||||
|--------|-----------------|------------|
|
||||
| MOV/FR reduction | Low embedding degree | Use curves with high embedding degree |
|
||||
| Anomalous curve attack | n = p | Ensure n ≠ p |
|
||||
| GHS attack | Extension field curves | Use prime field curves |
|
||||
|
||||
**secp256k1 is immune to all known curve-specific attacks**.
|
||||
|
||||
## Side-Channel Attacks
|
||||
|
||||
### Timing Attacks
|
||||
|
||||
**Vulnerability**: Execution time varies based on secret data.
|
||||
|
||||
**Examples**:
|
||||
- Conditional branches on secret bits
|
||||
- Early exit conditions
|
||||
- Variable-time modular operations
|
||||
|
||||
**Mitigations**:
|
||||
- Constant-time algorithms (Montgomery ladder)
|
||||
- Fixed execution paths
|
||||
- Dummy operations to equalize timing
|
||||
|
||||
### Power Analysis
|
||||
|
||||
**Simple Power Analysis (SPA)**: Single trace reveals operations.
|
||||
- Double-and-add visible as different power signatures
|
||||
- Mitigation: Montgomery ladder (uniform operations)
|
||||
|
||||
**Differential Power Analysis (DPA)**: Statistical analysis of many traces.
|
||||
- Mitigation: Point blinding, scalar blinding
|
||||
|
||||
### Cache Attacks
|
||||
|
||||
**FLUSH+RELOAD Attack**:
|
||||
```
|
||||
1. Attacker flushes cache line containing lookup table
|
||||
2. Victim performs table lookup based on secret
|
||||
3. Attacker measures reload time to determine which entry was accessed
|
||||
```
|
||||
|
||||
**Mitigations**:
|
||||
- Avoid secret-dependent table lookups
|
||||
- Use constant-time table access patterns
|
||||
- Scatter tables to prevent cache line sharing
|
||||
|
||||
### Electromagnetic (EM) Attacks
|
||||
|
||||
Similar to power analysis but captures electromagnetic emissions.
|
||||
|
||||
**Mitigations**:
|
||||
- Shielding
|
||||
- Same algorithmic protections as power analysis
|
||||
|
||||
## Implementation Vulnerabilities
|
||||
|
||||
### k-Reuse in ECDSA
|
||||
|
||||
**The Sony PS3 Hack (2010)**:
|
||||
|
||||
If the same k is used for two signatures (r₁, s₁) and (r₂, s₂) on messages m₁ and m₂:
|
||||
|
||||
```
|
||||
s₁ = k⁻¹(e₁ + rd) mod n
|
||||
s₂ = k⁻¹(e₂ + rd) mod n
|
||||
|
||||
Since k is the same:
|
||||
s₁ - s₂ = k⁻¹(e₁ - e₂) mod n
|
||||
k = (e₁ - e₂)(s₁ - s₂)⁻¹ mod n
|
||||
|
||||
Once k is known:
|
||||
d = (s₁k - e₁)r⁻¹ mod n
|
||||
```
|
||||
|
||||
**Mitigation**: Use deterministic k (RFC 6979).
|
||||
|
||||
### Weak Random k
|
||||
|
||||
Even with unique k values, if the RNG is biased:
|
||||
- Lattice-based attacks can recover private key
|
||||
- Only ~1% bias in k can be exploitable with enough signatures
|
||||
|
||||
**Mitigations**:
|
||||
- Use cryptographically secure RNG
|
||||
- Use deterministic k (RFC 6979)
|
||||
- Verify k is in valid range [1, n-1]
|
||||
|
||||
### Invalid Curve Attacks
|
||||
|
||||
**Attack**: Attacker provides point not on the curve.
|
||||
- Point may be on a weaker curve
|
||||
- Operations may leak information
|
||||
|
||||
**Mitigation**: Always validate points are on curve:
|
||||
```
|
||||
Verify: y² ≡ x³ + ax + b (mod p)
|
||||
```
|
||||
|
||||
### Small Subgroup Attacks
|
||||
|
||||
**Attack**: If cofactor h > 1, points of small order exist.
|
||||
- Attacker sends point of small order
|
||||
- Response reveals private key mod (small order)
|
||||
|
||||
**Mitigation**:
|
||||
- Use curves with cofactor 1 (secp256k1 has h = 1)
|
||||
- Multiply received points by cofactor
|
||||
- Validate point order
|
||||
|
||||
### Fault Attacks
|
||||
|
||||
**Attack**: Induce computational errors (voltage glitches, radiation).
|
||||
- Corrupted intermediate values may leak information
|
||||
- Differential fault analysis can recover keys
|
||||
|
||||
**Mitigations**:
|
||||
- Redundant computations with comparison
|
||||
- Verify final results
|
||||
- Hardware protections
|
||||
|
||||
## Signature Malleability
|
||||
|
||||
### ECDSA Malleability
|
||||
|
||||
Given valid signature (r, s), signature (r, n - s) is also valid for the same message.
|
||||
|
||||
**Impact**: Transaction ID malleability (historical Bitcoin issue)
|
||||
|
||||
**Mitigation**: Enforce low-S normalization:
|
||||
```
|
||||
if s > n/2:
|
||||
s = n - s
|
||||
```
|
||||
|
||||
### Schnorr Non-Malleability
|
||||
|
||||
BIP-340 Schnorr signatures are non-malleable by design:
|
||||
- Use x-only public keys
|
||||
- Deterministic nonce derivation
|
||||
|
||||
## Quantum Threats
|
||||
|
||||
### Shor's Algorithm
|
||||
|
||||
**Threat**: Polynomial-time discrete log on quantum computers.
|
||||
- Requires ~1500-2000 logical qubits for secp256k1
|
||||
- Current quantum computers: <100 noisy qubits
|
||||
|
||||
**Timeline**: Estimated 10-20+ years for cryptographically relevant quantum computers.
|
||||
|
||||
### Migration Strategy
|
||||
|
||||
1. **Monitor**: Track quantum computing progress
|
||||
2. **Prepare**: Develop post-quantum alternatives
|
||||
3. **Hybrid**: Use classical + post-quantum in transition
|
||||
4. **Migrate**: Full transition when necessary
|
||||
|
||||
### Post-Quantum Alternatives
|
||||
|
||||
- Lattice-based signatures (CRYSTALS-Dilithium)
|
||||
- Hash-based signatures (SPHINCS+)
|
||||
- Code-based cryptography
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Key Generation
|
||||
|
||||
```
|
||||
DO:
|
||||
- Use cryptographically secure RNG
|
||||
- Validate private key is in [1, n-1]
|
||||
- Verify public key is on curve
|
||||
- Verify public key is not point at infinity
|
||||
|
||||
DON'T:
|
||||
- Use predictable seeds
|
||||
- Use truncated random values
|
||||
- Skip validation
|
||||
```
|
||||
|
||||
### Signature Generation
|
||||
|
||||
```
|
||||
DO:
|
||||
- Use RFC 6979 for deterministic k
|
||||
- Validate all inputs
|
||||
- Use constant-time operations
|
||||
- Clear sensitive memory after use
|
||||
|
||||
DON'T:
|
||||
- Reuse k values
|
||||
- Use weak/biased RNG
|
||||
- Skip low-S normalization (ECDSA)
|
||||
```
|
||||
|
||||
### Signature Verification
|
||||
|
||||
```
|
||||
DO:
|
||||
- Validate r, s are in [1, n-1]
|
||||
- Validate public key is on curve
|
||||
- Validate public key is not infinity
|
||||
- Use batch verification when possible
|
||||
|
||||
DON'T:
|
||||
- Skip any validation steps
|
||||
- Accept malformed signatures
|
||||
```
|
||||
|
||||
### Public Key Handling
|
||||
|
||||
```
|
||||
DO:
|
||||
- Validate received points are on curve
|
||||
- Check point is not infinity
|
||||
- Prefer compressed format for storage
|
||||
|
||||
DON'T:
|
||||
- Accept unvalidated points
|
||||
- Skip curve membership check
|
||||
```
|
||||
|
||||
## Security Checklist
|
||||
|
||||
### Implementation Review
|
||||
|
||||
- [ ] All scalar multiplications are constant-time
|
||||
- [ ] No secret-dependent branches
|
||||
- [ ] No secret-indexed table lookups
|
||||
- [ ] Memory is zeroized after use
|
||||
- [ ] Random k uses CSPRNG or RFC 6979
|
||||
- [ ] All received points are validated
|
||||
- [ ] Private keys are in valid range
|
||||
- [ ] Signatures use low-S normalization
|
||||
|
||||
### Operational Security
|
||||
|
||||
- [ ] Private keys stored securely (HSM, secure enclave)
|
||||
- [ ] Key derivation uses proper KDF
|
||||
- [ ] Backups are encrypted
|
||||
- [ ] Key rotation policy exists
|
||||
- [ ] Audit logging enabled
|
||||
- [ ] Incident response plan exists
|
||||
|
||||
## Security Levels Comparison
|
||||
|
||||
| Curve | Bits | Symmetric Equivalent | RSA Equivalent |
|
||||
|-------|------|---------------------|----------------|
|
||||
| secp192r1 | 192 | 96 | 1536 |
|
||||
| secp224r1 | 224 | 112 | 2048 |
|
||||
| secp256k1 | 256 | 128 | 3072 |
|
||||
| secp384r1 | 384 | 192 | 7680 |
|
||||
| secp521r1 | 521 | 256 | 15360 |
|
||||
|
||||
## References
|
||||
|
||||
- NIST SP 800-57: Recommendation for Key Management
|
||||
- SEC 1: Elliptic Curve Cryptography
|
||||
- RFC 6979: Deterministic Usage of DSA and ECDSA
|
||||
- BIP-340: Schnorr Signatures for secp256k1
|
||||
- SafeCurves: Choosing Safe Curves for Elliptic-Curve Cryptography
|
||||
96
CLAUDE.md
96
CLAUDE.md
@@ -8,11 +8,12 @@ ORLY is a high-performance Nostr relay written in Go, designed for personal rela
|
||||
|
||||
**Key Technologies:**
|
||||
- **Language**: Go 1.25.3+
|
||||
- **Database**: Badger v4 (embedded key-value store) or DGraph (distributed graph database)
|
||||
- **Database**: Badger v4 (embedded), DGraph (distributed graph), or Neo4j (social graph)
|
||||
- **Cryptography**: Custom p8k library using purego for secp256k1 operations (no CGO)
|
||||
- **Web UI**: Svelte frontend embedded in the binary
|
||||
- **WebSocket**: gorilla/websocket for Nostr protocol
|
||||
- **Performance**: SIMD-accelerated SHA256 and hex encoding, query result caching with zstd compression
|
||||
- **Social Graph**: Neo4j backend with Web of Trust (WoT) extensions for trust metrics
|
||||
|
||||
## Build Commands
|
||||
|
||||
@@ -158,6 +159,19 @@ export ORLY_QUERY_CACHE_MAX_AGE=5m # Cache expiry time
|
||||
export ORLY_DB_BLOCK_CACHE_MB=512 # Block cache size
|
||||
export ORLY_DB_INDEX_CACHE_MB=256 # Index cache size
|
||||
export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
|
||||
# Directory Spider (metadata sync from other relays)
|
||||
export ORLY_DIRECTORY_SPIDER=true # Enable directory spider
|
||||
export ORLY_DIRECTORY_SPIDER_INTERVAL=24h # How often to run
|
||||
export ORLY_DIRECTORY_SPIDER_HOPS=3 # Max hops for relay discovery
|
||||
|
||||
# NIP-43 Relay Access Metadata
|
||||
export ORLY_NIP43_ENABLED=true # Enable invite system
|
||||
export ORLY_NIP43_INVITE_EXPIRY=24h # Invite code validity
|
||||
|
||||
# Authentication modes
|
||||
export ORLY_AUTH_REQUIRED=false # Require auth for all requests
|
||||
export ORLY_AUTH_TO_WRITE=false # Require auth only for writes
|
||||
```
|
||||
|
||||
## Code Architecture
|
||||
@@ -185,7 +199,7 @@ export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
|
||||
**`pkg/database/`** - Database abstraction layer with multiple backend support
|
||||
- `interface.go` - Database interface definition for pluggable backends
|
||||
- `factory.go` - Database backend selection (Badger or DGraph)
|
||||
- `factory.go` - Database backend selection (Badger, DGraph, or Neo4j)
|
||||
- `database.go` - Badger implementation with cache tuning and query cache
|
||||
- `save-event.go` - Event storage with index updates
|
||||
- `query-events.go` - Main query execution engine with filter normalization
|
||||
@@ -196,6 +210,15 @@ export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
- `identity.go` - Relay identity key management
|
||||
- `migrations.go` - Database schema migration runner
|
||||
|
||||
**`pkg/neo4j/`** - Neo4j graph database backend with social graph support
|
||||
- `neo4j.go` - Main database implementation
|
||||
- `schema.go` - Graph schema and index definitions (includes WoT extensions)
|
||||
- `query-events.go` - REQ filter to Cypher translation
|
||||
- `save-event.go` - Event storage with relationship creation
|
||||
- `social-event-processor.go` - Processes kinds 0, 3, 1984, 10000 for social graph
|
||||
- `WOT_SPEC.md` - Web of Trust data model specification (NostrUser nodes, trust metrics)
|
||||
- `MODIFYING_SCHEMA.md` - Guide for schema modifications
|
||||
|
||||
**`pkg/protocol/`** - Nostr protocol implementation
|
||||
- `ws/` - WebSocket message framing and parsing
|
||||
- `auth/` - NIP-42 authentication challenge/response
|
||||
@@ -231,6 +254,9 @@ export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
**`pkg/policy/`** - Event filtering and validation policies
|
||||
- Policy configuration loaded from `~/.config/ORLY/policy.json`
|
||||
- Per-kind size limits, age restrictions, custom scripts
|
||||
- **Write-Only Validation**: Size, age, tag, and expiry validations apply ONLY to write operations
|
||||
- **Read-Only Filtering**: `read_allow`, `read_deny`, `privileged` apply ONLY to read operations
|
||||
- See `docs/POLICY_CONFIGURATION_REFERENCE.md` for authoritative read vs write applicability
|
||||
- **Dynamic Policy Hot Reload via Kind 12345 Events:**
|
||||
- Policy admins can update policy configuration without relay restart
|
||||
- Kind 12345 events contain JSON policy in content field
|
||||
@@ -239,12 +265,16 @@ export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
- Policy admin follow lists (kind 3) trigger immediate cache refresh
|
||||
- `WriteAllowFollows` rule grants both read+write access to admin follows
|
||||
- Tag validation supports regex patterns per tag type
|
||||
- **New Policy Rule Fields:**
|
||||
- **Policy Rule Fields:**
|
||||
- `max_expiry_duration`: ISO-8601 duration format (e.g., "P7D", "PT1H30M") for event expiry limits
|
||||
- `protected_required`: Requires NIP-70 protected events (must have "-" tag)
|
||||
- `identifier_regex`: Regex pattern for validating "d" tag identifiers
|
||||
- `follows_whitelist_admins`: Per-rule admin pubkeys whose follows are whitelisted
|
||||
- `write_allow` / `write_deny`: Pubkey whitelist/blacklist for writing (write-only)
|
||||
- `read_allow` / `read_deny`: Pubkey whitelist/blacklist for reading (read-only)
|
||||
- `privileged`: Party-involved access control (read-only)
|
||||
- See `docs/POLICY_USAGE_GUIDE.md` for configuration examples
|
||||
- See `pkg/policy/README.md` for quick reference
|
||||
|
||||
**`pkg/sync/`** - Distributed synchronization
|
||||
- `cluster_manager.go` - Active replication between relay peers
|
||||
@@ -254,6 +284,12 @@ export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
**`pkg/spider/`** - Event syncing from other relays
|
||||
- `spider.go` - Spider manager for "follows" mode
|
||||
- Fetches events from admin relays for followed pubkeys
|
||||
- **Directory Spider** (`directory.go`):
|
||||
- Discovers relays by crawling kind 10002 (relay list) events
|
||||
- Expands outward from seed pubkeys (whitelisted users) via hop distance
|
||||
- Fetches metadata events (kinds 0, 3, 10000, 10002) from discovered relays
|
||||
- Self-detection prevents querying own relay
|
||||
- Configurable interval and max hops via `ORLY_DIRECTORY_SPIDER_*` env vars
|
||||
|
||||
**`pkg/utils/`** - Shared utilities
|
||||
- `atomic/` - Extended atomic operations
|
||||
@@ -287,6 +323,11 @@ export ORLY_INLINE_EVENT_THRESHOLD=1024 # Inline storage threshold (bytes)
|
||||
- Supports multiple backends via `ORLY_DB_TYPE` environment variable
|
||||
- **Badger** (default): Embedded key-value store with custom indexing, ideal for single-instance deployments
|
||||
- **DGraph**: Distributed graph database for larger, multi-node deployments
|
||||
- **Neo4j**: Graph database with social graph and Web of Trust (WoT) extensions
|
||||
- Processes kinds 0 (profile), 3 (contacts), 1984 (reports), 10000 (mute list) for social graph
|
||||
- NostrUser nodes with trust metrics (influence, PageRank)
|
||||
- FOLLOWS, MUTES, REPORTS relationships for WoT analysis
|
||||
- See `pkg/neo4j/WOT_SPEC.md` for full schema specification
|
||||
- Backend selected via factory pattern in `pkg/database/factory.go`
|
||||
- All backends implement the same `Database` interface defined in `pkg/database/interface.go`
|
||||
|
||||
@@ -538,3 +579,52 @@ Files modified:
|
||||
```
|
||||
3. GitHub Actions workflow builds binaries for multiple platforms
|
||||
4. Release created automatically with binaries and checksums
|
||||
|
||||
## Recent Features (v0.31.x)
|
||||
|
||||
### Directory Spider
|
||||
The directory spider (`pkg/spider/directory.go`) automatically discovers and syncs metadata from other relays:
|
||||
- Crawls kind 10002 (relay list) events to discover relays
|
||||
- Expands outward from seed pubkeys (whitelisted users) via configurable hop distance
|
||||
- Fetches essential metadata events (kinds 0, 3, 10000, 10002)
|
||||
- Self-detection prevents querying own relay
|
||||
- Enable with `ORLY_DIRECTORY_SPIDER=true`
|
||||
|
||||
### Neo4j Social Graph Backend
|
||||
The Neo4j backend (`pkg/neo4j/`) includes Web of Trust (WoT) extensions:
|
||||
- **Social Event Processor**: Handles kinds 0, 3, 1984, 10000 for social graph management
|
||||
- **NostrUser nodes**: Store profile data and trust metrics (influence, PageRank)
|
||||
- **Relationships**: FOLLOWS, MUTES, REPORTS for social graph analysis
|
||||
- **WoT Schema**: See `pkg/neo4j/WOT_SPEC.md` for full specification
|
||||
- **Schema Modifications**: See `pkg/neo4j/MODIFYING_SCHEMA.md` for how to update
|
||||
|
||||
### Policy System Enhancements
|
||||
- **Write-Only Validation**: Size, age, tag validations apply ONLY to writes
|
||||
- **Read-Only Filtering**: `read_allow`, `read_deny`, `privileged` apply ONLY to reads
|
||||
- **Scripts**: Policy scripts execute ONLY for write operations
|
||||
- **Reference Documentation**: `docs/POLICY_CONFIGURATION_REFERENCE.md` provides authoritative read vs write applicability
|
||||
- See also: `pkg/policy/README.md` for quick reference
|
||||
|
||||
### Authentication Modes
|
||||
- `ORLY_AUTH_REQUIRED=true`: Require authentication for ALL requests
|
||||
- `ORLY_AUTH_TO_WRITE=true`: Require authentication only for writes (allow anonymous reads)
|
||||
|
||||
### NIP-43 Relay Access Metadata
|
||||
Invite-based access control system:
|
||||
- `ORLY_NIP43_ENABLED=true`: Enable invite system
|
||||
- Publishes kind 8000/8001 events for member changes
|
||||
- Publishes kind 13534 membership list events
|
||||
- Configurable invite expiry via `ORLY_NIP43_INVITE_EXPIRY`
|
||||
|
||||
## Documentation Index
|
||||
|
||||
| Document | Purpose |
|
||||
|----------|---------|
|
||||
| `docs/POLICY_CONFIGURATION_REFERENCE.md` | Authoritative policy config reference with read/write applicability |
|
||||
| `docs/POLICY_USAGE_GUIDE.md` | Comprehensive policy system user guide |
|
||||
| `pkg/policy/README.md` | Policy system quick reference |
|
||||
| `pkg/neo4j/README.md` | Neo4j backend overview |
|
||||
| `pkg/neo4j/WOT_SPEC.md` | Web of Trust schema specification |
|
||||
| `pkg/neo4j/MODIFYING_SCHEMA.md` | How to modify Neo4j schema |
|
||||
| `pkg/neo4j/TESTING.md` | Neo4j testing guide |
|
||||
| `readme.adoc` | Project README with feature overview |
|
||||
|
||||
615
docs/POLICY_CONFIGURATION_REFERENCE.md
Normal file
615
docs/POLICY_CONFIGURATION_REFERENCE.md
Normal file
@@ -0,0 +1,615 @@
|
||||
# ORLY Policy Configuration Reference
|
||||
|
||||
This document provides a definitive reference for all policy configuration options and when each rule applies. Use this as the authoritative source for understanding policy behavior.
|
||||
|
||||
## Quick Reference: Read vs Write Applicability
|
||||
|
||||
| Rule Field | Write (EVENT) | Read (REQ) | Notes |
|
||||
|------------|:-------------:|:----------:|-------|
|
||||
| `size_limit` | ✅ | ❌ | Validates incoming events only |
|
||||
| `content_limit` | ✅ | ❌ | Validates incoming events only |
|
||||
| `max_age_of_event` | ✅ | ❌ | Prevents replay attacks |
|
||||
| `max_age_event_in_future` | ✅ | ❌ | Prevents future-dated events |
|
||||
| `max_expiry_duration` | ✅ | ❌ | Requires expiration tag |
|
||||
| `must_have_tags` | ✅ | ❌ | Validates required tags |
|
||||
| `protected_required` | ✅ | ❌ | Requires NIP-70 "-" tag |
|
||||
| `identifier_regex` | ✅ | ❌ | Validates "d" tag format |
|
||||
| `tag_validation` | ✅ | ❌ | Validates tag values with regex |
|
||||
| `write_allow` | ✅ | ❌ | Pubkey whitelist for writing |
|
||||
| `write_deny` | ✅ | ❌ | Pubkey blacklist for writing |
|
||||
| `read_allow` | ❌ | ✅ | Pubkey whitelist for reading |
|
||||
| `read_deny` | ❌ | ✅ | Pubkey blacklist for reading |
|
||||
| `privileged` | ❌ | ✅ | Party-involved access control |
|
||||
| `write_allow_follows` | ✅ | ✅ | Grants **both** read AND write |
|
||||
| `follows_whitelist_admins` | ✅ | ✅ | Grants **both** read AND write |
|
||||
| `script` | ✅ | ❌ | Scripts only run for writes |
|
||||
|
||||
---
|
||||
|
||||
## Core Principle: Validation vs Filtering
|
||||
|
||||
The policy system has two distinct modes of operation:
|
||||
|
||||
### Write Operations (EVENT messages)
|
||||
- **Purpose**: Validate and accept/reject incoming events
|
||||
- **All rules apply** except `read_allow`, `read_deny`, and `privileged`
|
||||
- Events are checked **before storage**
|
||||
- Rejected events are never stored
|
||||
|
||||
### Read Operations (REQ messages)
|
||||
- **Purpose**: Filter which stored events a user can retrieve
|
||||
- **Only access control rules apply**: `read_allow`, `read_deny`, `privileged`, `write_allow_follows`, `follows_whitelist_admins`
|
||||
- Validation rules (size, age, tags) do NOT apply
|
||||
- Scripts are NOT executed for reads
|
||||
- Filtering happens **after database query**
|
||||
|
||||
---
|
||||
|
||||
## Configuration Structure
|
||||
|
||||
```json
|
||||
{
|
||||
"default_policy": "allow|deny",
|
||||
"kind": {
|
||||
"whitelist": [1, 3, 7],
|
||||
"blacklist": [4, 42]
|
||||
},
|
||||
"owners": ["hex_pubkey_64_chars"],
|
||||
"policy_admins": ["hex_pubkey_64_chars"],
|
||||
"policy_follow_whitelist_enabled": true,
|
||||
"global": { /* Rule object */ },
|
||||
"rules": {
|
||||
"1": { /* Rule object for kind 1 */ },
|
||||
"30023": { /* Rule object for kind 30023 */ }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Top-Level Configuration Fields
|
||||
|
||||
### `default_policy`
|
||||
**Type**: `string`
|
||||
**Values**: `"allow"` (default) or `"deny"`
|
||||
**Applies to**: Both read and write
|
||||
|
||||
The fallback behavior when no specific rule makes a decision.
|
||||
|
||||
```json
|
||||
{
|
||||
"default_policy": "deny"
|
||||
}
|
||||
```
|
||||
|
||||
### `kind.whitelist` and `kind.blacklist`
|
||||
**Type**: `[]int`
|
||||
**Applies to**: Both read and write
|
||||
|
||||
Controls which event kinds are processed at all.
|
||||
|
||||
- **Whitelist** takes precedence: If present, ONLY whitelisted kinds are allowed
|
||||
- **Blacklist**: If no whitelist, these kinds are denied
|
||||
- **Neither**: Behavior depends on `default_policy` and whether rules exist
|
||||
|
||||
```json
|
||||
{
|
||||
"kind": {
|
||||
"whitelist": [0, 1, 3, 7, 30023]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### `owners`
|
||||
**Type**: `[]string` (64-character hex pubkeys)
|
||||
**Applies to**: Policy administration
|
||||
|
||||
Relay owners with full control. Merged with `ORLY_OWNERS` environment variable.
|
||||
|
||||
```json
|
||||
{
|
||||
"owners": ["4a93c5ac0c6f49d2c7e7a5b8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8"]
|
||||
}
|
||||
```
|
||||
|
||||
### `policy_admins`
|
||||
**Type**: `[]string` (64-character hex pubkeys)
|
||||
**Applies to**: Policy administration
|
||||
|
||||
Pubkeys that can update policy via kind 12345 events (with restrictions).
|
||||
|
||||
### `policy_follow_whitelist_enabled`
|
||||
**Type**: `boolean`
|
||||
**Default**: `false`
|
||||
**Applies to**: Both read and write (when `write_allow_follows` is true)
|
||||
|
||||
When enabled, allows `write_allow_follows` rules to grant access to policy admin follows.
|
||||
|
||||
---
|
||||
|
||||
## Rule Object Fields
|
||||
|
||||
Rules can be defined in `global` (applies to all events) or `rules[kind]` (applies to specific kind).
|
||||
|
||||
### Access Control Fields
|
||||
|
||||
#### `write_allow`
|
||||
**Type**: `[]string` (hex pubkeys)
|
||||
**Applies to**: Write only
|
||||
**Behavior**: Exclusive whitelist
|
||||
|
||||
When present with entries, ONLY these pubkeys can write events of this kind. All others are denied.
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"1": {
|
||||
"write_allow": ["pubkey1_hex", "pubkey2_hex"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Special case**: Empty array `[]` explicitly allows all writers.
|
||||
|
||||
#### `write_deny`
|
||||
**Type**: `[]string` (hex pubkeys)
|
||||
**Applies to**: Write only
|
||||
**Behavior**: Blacklist (highest priority)
|
||||
|
||||
These pubkeys cannot write events of this kind. **Checked before allow lists.**
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"1": {
|
||||
"write_deny": ["banned_pubkey_hex"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `read_allow`
|
||||
**Type**: `[]string` (hex pubkeys)
|
||||
**Applies to**: Read only
|
||||
**Behavior**: Exclusive whitelist (with OR logic for privileged)
|
||||
|
||||
When present with entries:
|
||||
- If `privileged: false`: ONLY these pubkeys can read
|
||||
- If `privileged: true`: These pubkeys OR parties involved can read
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"4": {
|
||||
"read_allow": ["trusted_pubkey_hex"],
|
||||
"privileged": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `read_deny`
|
||||
**Type**: `[]string` (hex pubkeys)
|
||||
**Applies to**: Read only
|
||||
**Behavior**: Blacklist (highest priority)
|
||||
|
||||
These pubkeys cannot read events of this kind. **Checked before allow lists.**
|
||||
|
||||
#### `privileged`
|
||||
**Type**: `boolean`
|
||||
**Default**: `false`
|
||||
**Applies to**: Read only
|
||||
|
||||
When `true`, events are only readable by "parties involved":
|
||||
- The event author (`event.pubkey`)
|
||||
- Users mentioned in `p` tags
|
||||
|
||||
**Interaction with `read_allow`**:
|
||||
- `read_allow` present + `privileged: true` = OR logic (in list OR party involved)
|
||||
- `read_allow` empty + `privileged: true` = Only parties involved
|
||||
- `privileged: true` alone = Only parties involved
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"4": {
|
||||
"description": "DMs - only sender and recipient can read",
|
||||
"privileged": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `write_allow_follows`
|
||||
**Type**: `boolean`
|
||||
**Default**: `false`
|
||||
**Applies to**: Both read AND write
|
||||
**Requires**: `policy_follow_whitelist_enabled: true` at top level
|
||||
|
||||
Grants **both read and write access** to pubkeys followed by policy admins.
|
||||
|
||||
> **Important**: Despite the name, this grants BOTH read and write access.
|
||||
|
||||
```json
|
||||
{
|
||||
"policy_follow_whitelist_enabled": true,
|
||||
"rules": {
|
||||
"1": {
|
||||
"write_allow_follows": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `follows_whitelist_admins`
|
||||
**Type**: `[]string` (hex pubkeys)
|
||||
**Applies to**: Both read AND write
|
||||
|
||||
Alternative to `write_allow_follows` that specifies which admin pubkeys' follows are whitelisted for this specific rule.
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"30023": {
|
||||
"follows_whitelist_admins": ["curator_pubkey_hex"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Validation Fields (Write-Only)
|
||||
|
||||
These fields validate incoming events and are **completely ignored for read operations**.
|
||||
|
||||
#### `size_limit`
|
||||
**Type**: `int64` (bytes)
|
||||
**Applies to**: Write only
|
||||
|
||||
Maximum total serialized event size.
|
||||
|
||||
```json
|
||||
{
|
||||
"global": {
|
||||
"size_limit": 100000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `content_limit`
|
||||
**Type**: `int64` (bytes)
|
||||
**Applies to**: Write only
|
||||
|
||||
Maximum size of the `content` field.
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"1": {
|
||||
"content_limit": 10000
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `max_age_of_event`
|
||||
**Type**: `int64` (seconds)
|
||||
**Applies to**: Write only
|
||||
|
||||
Maximum age of events. Events with `created_at` older than `now - max_age_of_event` are rejected.
|
||||
|
||||
```json
|
||||
{
|
||||
"global": {
|
||||
"max_age_of_event": 86400
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `max_age_event_in_future`
|
||||
**Type**: `int64` (seconds)
|
||||
**Applies to**: Write only
|
||||
|
||||
Maximum time events can be dated in the future. Events with `created_at` later than `now + max_age_event_in_future` are rejected.
|
||||
|
||||
```json
|
||||
{
|
||||
"global": {
|
||||
"max_age_event_in_future": 300
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `max_expiry_duration`
|
||||
**Type**: `string` (ISO-8601 duration)
|
||||
**Applies to**: Write only
|
||||
|
||||
Maximum allowed expiry time from event creation. Events **must** have an `expiration` tag when this is set.
|
||||
|
||||
**Format**: `P[n]Y[n]M[n]W[n]DT[n]H[n]M[n]S`
|
||||
|
||||
**Examples**:
|
||||
- `P7D` = 7 days
|
||||
- `PT1H` = 1 hour
|
||||
- `P1DT12H` = 1 day 12 hours
|
||||
- `PT30M` = 30 minutes
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"20": {
|
||||
"description": "Ephemeral events must expire within 24 hours",
|
||||
"max_expiry_duration": "P1D"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `must_have_tags`
|
||||
**Type**: `[]string` (tag names)
|
||||
**Applies to**: Write only
|
||||
|
||||
Required tags that must be present on the event.
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"1": {
|
||||
"must_have_tags": ["p", "e"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `protected_required`
|
||||
**Type**: `boolean`
|
||||
**Default**: `false`
|
||||
**Applies to**: Write only
|
||||
|
||||
Requires events to have a `-` tag (NIP-70 protected events).
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"4": {
|
||||
"protected_required": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `identifier_regex`
|
||||
**Type**: `string` (regex pattern)
|
||||
**Applies to**: Write only
|
||||
|
||||
Regex pattern that `d` tag values must match. Events **must** have a `d` tag when this is set.
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"30023": {
|
||||
"identifier_regex": "^[a-z0-9-]{1,64}$"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `tag_validation`
|
||||
**Type**: `map[string]string` (tag name → regex pattern)
|
||||
**Applies to**: Write only
|
||||
|
||||
Regex patterns for validating specific tag values. Only validates tags that are **present** on the event.
|
||||
|
||||
> **Note**: To require a tag to exist, use `must_have_tags`. `tag_validation` only validates format.
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"30023": {
|
||||
"tag_validation": {
|
||||
"t": "^[a-z0-9-]{1,32}$",
|
||||
"d": "^[a-z0-9-]+$"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Script Configuration
|
||||
|
||||
#### `script`
|
||||
**Type**: `string` (file path)
|
||||
**Applies to**: Write only
|
||||
|
||||
Path to a custom validation script. **Scripts are NOT executed for read operations.**
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"1": {
|
||||
"script": "/etc/orly/scripts/spam-filter.py"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Policy Evaluation Order
|
||||
|
||||
### For Write Operations
|
||||
|
||||
```
|
||||
1. Global Rule Check (all fields apply)
|
||||
├─ Universal constraints (size, tags, age, etc.)
|
||||
├─ write_deny check
|
||||
├─ write_allow_follows / follows_whitelist_admins check
|
||||
└─ write_allow check
|
||||
|
||||
2. Kind Filtering (whitelist/blacklist)
|
||||
|
||||
3. Kind-Specific Rule Check (same as global)
|
||||
├─ Universal constraints
|
||||
├─ write_deny check
|
||||
├─ write_allow_follows / follows_whitelist_admins check
|
||||
├─ write_allow check
|
||||
└─ Script execution (if configured)
|
||||
|
||||
4. Default Policy (if no rules matched)
|
||||
```
|
||||
|
||||
### For Read Operations
|
||||
|
||||
```
|
||||
1. Global Rule Check (access control only)
|
||||
├─ read_deny check
|
||||
├─ write_allow_follows / follows_whitelist_admins check
|
||||
├─ read_allow check
|
||||
└─ privileged check (party involved)
|
||||
|
||||
2. Kind Filtering (whitelist/blacklist)
|
||||
|
||||
3. Kind-Specific Rule Check (access control only)
|
||||
├─ read_deny check
|
||||
├─ write_allow_follows / follows_whitelist_admins check
|
||||
├─ read_allow + privileged (OR logic)
|
||||
└─ privileged-only check
|
||||
|
||||
4. Default Policy (if no rules matched)
|
||||
|
||||
NOTE: Scripts are NOT executed for read operations
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Common Configuration Patterns
|
||||
|
||||
### Private Relay (Whitelist Only)
|
||||
|
||||
```json
|
||||
{
|
||||
"default_policy": "deny",
|
||||
"global": {
|
||||
"write_allow": ["trusted_pubkey_1", "trusted_pubkey_2"],
|
||||
"read_allow": ["trusted_pubkey_1", "trusted_pubkey_2"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Open Relay with Spam Protection
|
||||
|
||||
```json
|
||||
{
|
||||
"default_policy": "allow",
|
||||
"global": {
|
||||
"size_limit": 100000,
|
||||
"max_age_of_event": 86400,
|
||||
"max_age_event_in_future": 300
|
||||
},
|
||||
"rules": {
|
||||
"1": {
|
||||
"script": "/etc/orly/scripts/spam-filter.sh"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Community Relay (Follows-Based)
|
||||
|
||||
```json
|
||||
{
|
||||
"default_policy": "deny",
|
||||
"policy_admins": ["community_admin_pubkey"],
|
||||
"policy_follow_whitelist_enabled": true,
|
||||
"global": {
|
||||
"write_allow_follows": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Encrypted DMs (Privileged Access)
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"4": {
|
||||
"description": "Encrypted DMs - only sender/recipient",
|
||||
"privileged": true,
|
||||
"protected_required": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Long-Form Content with Validation
|
||||
|
||||
```json
|
||||
{
|
||||
"rules": {
|
||||
"30023": {
|
||||
"description": "Long-form articles",
|
||||
"size_limit": 100000,
|
||||
"content_limit": 50000,
|
||||
"max_expiry_duration": "P30D",
|
||||
"identifier_regex": "^[a-z0-9-]{1,64}$",
|
||||
"tag_validation": {
|
||||
"t": "^[a-z0-9-]{1,32}$"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Important Behaviors
|
||||
|
||||
### Whitelist vs Blacklist Precedence
|
||||
|
||||
1. **Deny lists** (`write_deny`, `read_deny`) are checked **first** and have highest priority
|
||||
2. **Allow lists** are exclusive when populated - ONLY listed pubkeys are allowed
|
||||
3. **Deny-only configuration**: If only deny list exists (no allow list), all non-denied pubkeys are allowed
|
||||
|
||||
### Empty Arrays vs Null
|
||||
|
||||
- `[]` (empty array explicitly set) = Allow all
|
||||
- `null` or field omitted = No list configured, use other rules
|
||||
|
||||
### Global Rules Are Additive
|
||||
|
||||
Global rules are always evaluated **in addition to** kind-specific rules. They cannot be overridden at the kind level.
|
||||
|
||||
### Implicit Kind Whitelist
|
||||
|
||||
When rules are defined but no explicit `kind.whitelist`:
|
||||
- If `default_policy: "allow"`: All kinds allowed
|
||||
- If `default_policy: "deny"` or unset: Only kinds with rules allowed
|
||||
|
||||
---
|
||||
|
||||
## Debugging Policy Issues
|
||||
|
||||
Enable debug logging to see policy decisions:
|
||||
|
||||
```bash
|
||||
export ORLY_LOG_LEVEL=debug
|
||||
```
|
||||
|
||||
Log messages include:
|
||||
- Policy evaluation steps
|
||||
- Rule matching
|
||||
- Access decisions with reasons
|
||||
|
||||
---
|
||||
|
||||
## Source Code Reference
|
||||
|
||||
- Policy struct definition: `pkg/policy/policy.go:75-144` (Rule struct)
|
||||
- Policy struct definition: `pkg/policy/policy.go:380-412` (P struct)
|
||||
- Check evaluation: `pkg/policy/policy.go:1260-1595` (checkRulePolicy)
|
||||
- Write handler: `app/handle-event.go:114-138`
|
||||
- Read handler: `app/handle-req.go:420-438`
|
||||
@@ -134,6 +134,10 @@ CREATE (e)-[:AUTHORED_BY]->(a)
|
||||
eTagIndex := 0
|
||||
pTagIndex := 0
|
||||
|
||||
// Track if we need to add WITH clause before OPTIONAL MATCH
|
||||
// This is required because Cypher doesn't allow MATCH after CREATE without WITH
|
||||
needsWithClause := true
|
||||
|
||||
// Only process tags if they exist
|
||||
if ev.Tags != nil {
|
||||
for _, tagItem := range *ev.Tags {
|
||||
@@ -150,6 +154,15 @@ CREATE (e)-[:AUTHORED_BY]->(a)
|
||||
paramName := fmt.Sprintf("eTag_%d", eTagIndex)
|
||||
params[paramName] = tagValue
|
||||
|
||||
// Add WITH clause before first OPTIONAL MATCH to transition from CREATE to MATCH
|
||||
if needsWithClause {
|
||||
cypher += `
|
||||
// Carry forward event and author nodes for tag processing
|
||||
WITH e, a
|
||||
`
|
||||
needsWithClause = false
|
||||
}
|
||||
|
||||
cypher += fmt.Sprintf(`
|
||||
// Reference to event (e-tag)
|
||||
OPTIONAL MATCH (ref%d:Event {id: $%s})
|
||||
|
||||
824
pkg/neo4j/save-event_test.go
Normal file
824
pkg/neo4j/save-event_test.go
Normal file
@@ -0,0 +1,824 @@
|
||||
package neo4j
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.mleku.dev/mleku/nostr/encoders/event"
|
||||
"git.mleku.dev/mleku/nostr/encoders/hex"
|
||||
"git.mleku.dev/mleku/nostr/encoders/tag"
|
||||
"git.mleku.dev/mleku/nostr/encoders/timestamp"
|
||||
"git.mleku.dev/mleku/nostr/interfaces/signer/p8k"
|
||||
)
|
||||
|
||||
// TestCypherQueryGeneration_WithClause is a unit test that validates the WITH clause fix
|
||||
// without requiring a Neo4j instance. This test verifies the generated Cypher string
|
||||
// has correct syntax for different tag combinations.
|
||||
func TestCypherQueryGeneration_WithClause(t *testing.T) {
|
||||
// Create a mock N struct - we only need it to call buildEventCreationCypher
|
||||
// No actual Neo4j connection is needed for this unit test
|
||||
n := &N{}
|
||||
|
||||
// Generate test keypair
|
||||
signer, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := signer.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
tags *tag.S
|
||||
expectWithClause bool
|
||||
expectOptionalMatch bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "NoTags",
|
||||
tags: nil,
|
||||
expectWithClause: false,
|
||||
expectOptionalMatch: false,
|
||||
description: "Event without tags",
|
||||
},
|
||||
{
|
||||
name: "OnlyPTags_NoWithNeeded",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000001"),
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000002"),
|
||||
),
|
||||
expectWithClause: false,
|
||||
expectOptionalMatch: false,
|
||||
description: "p-tags use MERGE (not OPTIONAL MATCH), no WITH needed",
|
||||
},
|
||||
{
|
||||
name: "OnlyETags_WithRequired",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("e", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
|
||||
tag.NewFromAny("e", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
|
||||
),
|
||||
expectWithClause: true,
|
||||
expectOptionalMatch: true,
|
||||
description: "e-tags use OPTIONAL MATCH which requires WITH clause after CREATE",
|
||||
},
|
||||
{
|
||||
name: "ETagBeforePTag",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("e", "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"),
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000003"),
|
||||
),
|
||||
expectWithClause: true,
|
||||
expectOptionalMatch: true,
|
||||
description: "e-tag appearing first triggers WITH clause",
|
||||
},
|
||||
{
|
||||
name: "PTagBeforeETag",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000004"),
|
||||
tag.NewFromAny("e", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"),
|
||||
),
|
||||
expectWithClause: true,
|
||||
expectOptionalMatch: true,
|
||||
description: "WITH clause needed even when p-tag comes before e-tag",
|
||||
},
|
||||
{
|
||||
name: "GenericTagsBeforeETag",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("t", "nostr"),
|
||||
tag.NewFromAny("r", "https://example.com"),
|
||||
tag.NewFromAny("e", "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"),
|
||||
),
|
||||
expectWithClause: true,
|
||||
expectOptionalMatch: true,
|
||||
description: "WITH clause needed when e-tag follows generic tags",
|
||||
},
|
||||
{
|
||||
name: "OnlyGenericTags",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("t", "bitcoin"),
|
||||
tag.NewFromAny("d", "identifier"),
|
||||
tag.NewFromAny("r", "wss://relay.example.com"),
|
||||
),
|
||||
expectWithClause: false,
|
||||
expectOptionalMatch: false,
|
||||
description: "Generic tags use MERGE, no WITH needed",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Create test event
|
||||
ev := event.New()
|
||||
ev.Pubkey = signer.Pub()
|
||||
ev.CreatedAt = timestamp.Now().V
|
||||
ev.Kind = 1
|
||||
ev.Content = []byte(fmt.Sprintf("Test content for %s", tt.name))
|
||||
ev.Tags = tt.tags
|
||||
|
||||
if err := ev.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign event: %v", err)
|
||||
}
|
||||
|
||||
// Generate Cypher query
|
||||
cypher, params := n.buildEventCreationCypher(ev, 12345)
|
||||
|
||||
// Validate WITH clause presence
|
||||
hasWithClause := strings.Contains(cypher, "WITH e, a")
|
||||
if tt.expectWithClause && !hasWithClause {
|
||||
t.Errorf("%s: expected WITH clause but none found in Cypher:\n%s", tt.description, cypher)
|
||||
}
|
||||
if !tt.expectWithClause && hasWithClause {
|
||||
t.Errorf("%s: unexpected WITH clause in Cypher:\n%s", tt.description, cypher)
|
||||
}
|
||||
|
||||
// Validate OPTIONAL MATCH presence
|
||||
hasOptionalMatch := strings.Contains(cypher, "OPTIONAL MATCH")
|
||||
if tt.expectOptionalMatch && !hasOptionalMatch {
|
||||
t.Errorf("%s: expected OPTIONAL MATCH but none found", tt.description)
|
||||
}
|
||||
if !tt.expectOptionalMatch && hasOptionalMatch {
|
||||
t.Errorf("%s: unexpected OPTIONAL MATCH found", tt.description)
|
||||
}
|
||||
|
||||
// Validate WITH clause comes BEFORE first OPTIONAL MATCH (if both present)
|
||||
if hasWithClause && hasOptionalMatch {
|
||||
withIndex := strings.Index(cypher, "WITH e, a")
|
||||
optionalIndex := strings.Index(cypher, "OPTIONAL MATCH")
|
||||
if withIndex > optionalIndex {
|
||||
t.Errorf("%s: WITH clause must come BEFORE OPTIONAL MATCH.\nWITH at %d, OPTIONAL MATCH at %d\nCypher:\n%s",
|
||||
tt.description, withIndex, optionalIndex, cypher)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate parameters are set
|
||||
if params == nil {
|
||||
t.Error("params should not be nil")
|
||||
}
|
||||
|
||||
// Validate basic required params exist
|
||||
if _, ok := params["eventId"]; !ok {
|
||||
t.Error("params should contain eventId")
|
||||
}
|
||||
if _, ok := params["serial"]; !ok {
|
||||
t.Error("params should contain serial")
|
||||
}
|
||||
|
||||
t.Logf("✓ %s: WITH=%v, OPTIONAL_MATCH=%v", tt.name, hasWithClause, hasOptionalMatch)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCypherQueryGeneration_MultipleETags verifies WITH clause is added exactly once
|
||||
// even with multiple e-tags.
|
||||
func TestCypherQueryGeneration_MultipleETags(t *testing.T) {
|
||||
n := &N{}
|
||||
|
||||
signer, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := signer.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
// Create event with many e-tags
|
||||
manyETags := tag.NewS()
|
||||
for i := 0; i < 10; i++ {
|
||||
manyETags.Append(tag.NewFromAny("e", fmt.Sprintf("%064x", i)))
|
||||
}
|
||||
|
||||
ev := event.New()
|
||||
ev.Pubkey = signer.Pub()
|
||||
ev.CreatedAt = timestamp.Now().V
|
||||
ev.Kind = 1
|
||||
ev.Content = []byte("Event with many e-tags")
|
||||
ev.Tags = manyETags
|
||||
|
||||
if err := ev.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign event: %v", err)
|
||||
}
|
||||
|
||||
cypher, _ := n.buildEventCreationCypher(ev, 1)
|
||||
|
||||
// Count WITH clauses - should be exactly 1
|
||||
withCount := strings.Count(cypher, "WITH e, a")
|
||||
if withCount != 1 {
|
||||
t.Errorf("Expected exactly 1 WITH clause, found %d\nCypher:\n%s", withCount, cypher)
|
||||
}
|
||||
|
||||
// Count OPTIONAL MATCH - should match number of e-tags
|
||||
optionalMatchCount := strings.Count(cypher, "OPTIONAL MATCH")
|
||||
if optionalMatchCount != 10 {
|
||||
t.Errorf("Expected 10 OPTIONAL MATCH statements (one per e-tag), found %d", optionalMatchCount)
|
||||
}
|
||||
|
||||
// Count FOREACH (which wraps the conditional relationship creation)
|
||||
foreachCount := strings.Count(cypher, "FOREACH")
|
||||
if foreachCount != 10 {
|
||||
t.Errorf("Expected 10 FOREACH blocks, found %d", foreachCount)
|
||||
}
|
||||
|
||||
t.Logf("✓ WITH clause added once, followed by %d OPTIONAL MATCH + FOREACH pairs", optionalMatchCount)
|
||||
}
|
||||
|
||||
// TestCypherQueryGeneration_CriticalBugScenario reproduces the exact bug scenario
|
||||
// that was fixed: CREATE followed by OPTIONAL MATCH without WITH clause.
|
||||
func TestCypherQueryGeneration_CriticalBugScenario(t *testing.T) {
|
||||
n := &N{}
|
||||
|
||||
signer, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := signer.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
// This is the exact scenario that caused the bug:
|
||||
// An event with just one e-tag should have:
|
||||
// 1. CREATE clause for the event
|
||||
// 2. WITH clause to carry forward variables
|
||||
// 3. OPTIONAL MATCH for the referenced event
|
||||
ev := event.New()
|
||||
ev.Pubkey = signer.Pub()
|
||||
ev.CreatedAt = timestamp.Now().V
|
||||
ev.Kind = 1
|
||||
ev.Content = []byte("Reply to an event")
|
||||
ev.Tags = tag.NewS(
|
||||
tag.NewFromAny("e", "1234567890123456789012345678901234567890123456789012345678901234"),
|
||||
)
|
||||
|
||||
if err := ev.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign event: %v", err)
|
||||
}
|
||||
|
||||
cypher, _ := n.buildEventCreationCypher(ev, 1)
|
||||
|
||||
// The critical validation: WITH must appear between CREATE and OPTIONAL MATCH
|
||||
createIndex := strings.Index(cypher, "CREATE (e)-[:AUTHORED_BY]->(a)")
|
||||
withIndex := strings.Index(cypher, "WITH e, a")
|
||||
optionalMatchIndex := strings.Index(cypher, "OPTIONAL MATCH")
|
||||
|
||||
if createIndex == -1 {
|
||||
t.Fatal("CREATE clause not found in Cypher")
|
||||
}
|
||||
if withIndex == -1 {
|
||||
t.Fatal("WITH clause not found in Cypher - THIS IS THE BUG!")
|
||||
}
|
||||
if optionalMatchIndex == -1 {
|
||||
t.Fatal("OPTIONAL MATCH not found in Cypher")
|
||||
}
|
||||
|
||||
// Validate order: CREATE < WITH < OPTIONAL MATCH
|
||||
if !(createIndex < withIndex && withIndex < optionalMatchIndex) {
|
||||
t.Errorf("Invalid clause ordering. Expected: CREATE (%d) < WITH (%d) < OPTIONAL MATCH (%d)\nCypher:\n%s",
|
||||
createIndex, withIndex, optionalMatchIndex, cypher)
|
||||
}
|
||||
|
||||
t.Log("✓ Critical bug scenario validated: WITH clause correctly placed between CREATE and OPTIONAL MATCH")
|
||||
}
|
||||
|
||||
// TestBuildEventCreationCypher_WithClause validates the WITH clause fix for Cypher queries.
|
||||
// The bug was that OPTIONAL MATCH cannot directly follow CREATE in Cypher - a WITH clause
|
||||
// is required to carry forward bound variables (e, a) from the CREATE to the MATCH.
|
||||
func TestBuildEventCreationCypher_WithClause(t *testing.T) {
|
||||
// Skip if Neo4j is not available
|
||||
neo4jURI := os.Getenv("ORLY_NEO4J_URI")
|
||||
if neo4jURI == "" {
|
||||
t.Skip("Skipping Neo4j test: ORLY_NEO4J_URI not set")
|
||||
}
|
||||
|
||||
// Create test database
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
tempDir := t.TempDir()
|
||||
db, err := New(ctx, cancel, tempDir, "debug")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Wait for database to be ready
|
||||
<-db.Ready()
|
||||
|
||||
// Wipe database to ensure clean state
|
||||
if err := db.Wipe(); err != nil {
|
||||
t.Fatalf("Failed to wipe database: %v", err)
|
||||
}
|
||||
|
||||
// Generate test keypair
|
||||
signer, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := signer.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
// Test cases for different tag combinations
|
||||
tests := []struct {
|
||||
name string
|
||||
tags *tag.S
|
||||
wantWithClause bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "NoTags",
|
||||
tags: nil,
|
||||
wantWithClause: false,
|
||||
description: "Event without tags should not have WITH clause",
|
||||
},
|
||||
{
|
||||
name: "OnlyPTags",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000001"),
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000002"),
|
||||
),
|
||||
wantWithClause: false,
|
||||
description: "Event with only p-tags (MERGE) should not have WITH clause",
|
||||
},
|
||||
{
|
||||
name: "OnlyETags",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("e", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
|
||||
tag.NewFromAny("e", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
|
||||
),
|
||||
wantWithClause: true,
|
||||
description: "Event with e-tags (OPTIONAL MATCH) MUST have WITH clause",
|
||||
},
|
||||
{
|
||||
name: "ETagFirst",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("e", "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"),
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000003"),
|
||||
),
|
||||
wantWithClause: true,
|
||||
description: "Event with e-tag first MUST have WITH clause before OPTIONAL MATCH",
|
||||
},
|
||||
{
|
||||
name: "PTagFirst",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000004"),
|
||||
tag.NewFromAny("e", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"),
|
||||
),
|
||||
wantWithClause: true,
|
||||
description: "Event with p-tag first still needs WITH clause before e-tag's OPTIONAL MATCH",
|
||||
},
|
||||
{
|
||||
name: "MixedTags",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("t", "nostr"),
|
||||
tag.NewFromAny("e", "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"),
|
||||
tag.NewFromAny("p", "0000000000000000000000000000000000000000000000000000000000000005"),
|
||||
tag.NewFromAny("r", "https://example.com"),
|
||||
),
|
||||
wantWithClause: true,
|
||||
description: "Mixed tags with e-tag requires WITH clause",
|
||||
},
|
||||
{
|
||||
name: "OnlyGenericTags",
|
||||
tags: tag.NewS(
|
||||
tag.NewFromAny("t", "bitcoin"),
|
||||
tag.NewFromAny("r", "wss://relay.example.com"),
|
||||
tag.NewFromAny("d", "identifier"),
|
||||
),
|
||||
wantWithClause: false,
|
||||
description: "Generic tags (MERGE) don't require WITH clause",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Create event
|
||||
ev := event.New()
|
||||
ev.Pubkey = signer.Pub()
|
||||
ev.CreatedAt = timestamp.Now().V
|
||||
ev.Kind = 1
|
||||
ev.Content = []byte(fmt.Sprintf("Test content for %s", tt.name))
|
||||
ev.Tags = tt.tags
|
||||
|
||||
if err := ev.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign event: %v", err)
|
||||
}
|
||||
|
||||
// Build Cypher query
|
||||
cypher, params := db.buildEventCreationCypher(ev, 1)
|
||||
|
||||
// Check if WITH clause is present
|
||||
hasWithClause := strings.Contains(cypher, "WITH e, a")
|
||||
|
||||
if tt.wantWithClause && !hasWithClause {
|
||||
t.Errorf("%s: expected WITH clause but none found.\nCypher:\n%s", tt.description, cypher)
|
||||
}
|
||||
if !tt.wantWithClause && hasWithClause {
|
||||
t.Errorf("%s: unexpected WITH clause found.\nCypher:\n%s", tt.description, cypher)
|
||||
}
|
||||
|
||||
// Verify Cypher syntax by executing it against Neo4j
|
||||
// This is the key test - invalid Cypher will fail here
|
||||
_, err := db.ExecuteWrite(ctx, cypher, params)
|
||||
if err != nil {
|
||||
t.Errorf("%s: Cypher query failed (invalid syntax): %v\nCypher:\n%s", tt.description, err, cypher)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSaveEvent_ETagReference tests that events with e-tags are saved correctly
|
||||
// and the REFERENCES relationships are created when the referenced event exists.
|
||||
func TestSaveEvent_ETagReference(t *testing.T) {
|
||||
neo4jURI := os.Getenv("ORLY_NEO4J_URI")
|
||||
if neo4jURI == "" {
|
||||
t.Skip("Skipping Neo4j test: ORLY_NEO4J_URI not set")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
tempDir := t.TempDir()
|
||||
db, err := New(ctx, cancel, tempDir, "debug")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
<-db.Ready()
|
||||
|
||||
if err := db.Wipe(); err != nil {
|
||||
t.Fatalf("Failed to wipe database: %v", err)
|
||||
}
|
||||
|
||||
// Generate keypairs
|
||||
alice, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := alice.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
bob, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := bob.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
// Create a root event from Alice
|
||||
rootEvent := event.New()
|
||||
rootEvent.Pubkey = alice.Pub()
|
||||
rootEvent.CreatedAt = timestamp.Now().V
|
||||
rootEvent.Kind = 1
|
||||
rootEvent.Content = []byte("This is the root event")
|
||||
|
||||
if err := rootEvent.Sign(alice); err != nil {
|
||||
t.Fatalf("Failed to sign root event: %v", err)
|
||||
}
|
||||
|
||||
// Save root event
|
||||
exists, err := db.SaveEvent(ctx, rootEvent)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to save root event: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("Root event should not exist yet")
|
||||
}
|
||||
|
||||
rootEventID := hex.Enc(rootEvent.ID[:])
|
||||
|
||||
// Create a reply from Bob that references the root event
|
||||
replyEvent := event.New()
|
||||
replyEvent.Pubkey = bob.Pub()
|
||||
replyEvent.CreatedAt = timestamp.Now().V + 1
|
||||
replyEvent.Kind = 1
|
||||
replyEvent.Content = []byte("This is a reply to Alice")
|
||||
replyEvent.Tags = tag.NewS(
|
||||
tag.NewFromAny("e", rootEventID, "", "root"),
|
||||
tag.NewFromAny("p", hex.Enc(alice.Pub())),
|
||||
)
|
||||
|
||||
if err := replyEvent.Sign(bob); err != nil {
|
||||
t.Fatalf("Failed to sign reply event: %v", err)
|
||||
}
|
||||
|
||||
// Save reply event - this exercises the WITH clause fix
|
||||
exists, err = db.SaveEvent(ctx, replyEvent)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to save reply event: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("Reply event should not exist yet")
|
||||
}
|
||||
|
||||
// Verify REFERENCES relationship was created
|
||||
cypher := `
|
||||
MATCH (reply:Event {id: $replyId})-[:REFERENCES]->(root:Event {id: $rootId})
|
||||
RETURN reply.id AS replyId, root.id AS rootId
|
||||
`
|
||||
params := map[string]any{
|
||||
"replyId": hex.Enc(replyEvent.ID[:]),
|
||||
"rootId": rootEventID,
|
||||
}
|
||||
|
||||
result, err := db.ExecuteRead(ctx, cypher, params)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to query REFERENCES relationship: %v", err)
|
||||
}
|
||||
|
||||
if !result.Next(ctx) {
|
||||
t.Error("Expected REFERENCES relationship between reply and root events")
|
||||
} else {
|
||||
record := result.Record()
|
||||
returnedReplyId := record.Values[0].(string)
|
||||
returnedRootId := record.Values[1].(string)
|
||||
t.Logf("✓ REFERENCES relationship verified: %s -> %s", returnedReplyId[:8], returnedRootId[:8])
|
||||
}
|
||||
|
||||
// Verify MENTIONS relationship was also created for the p-tag
|
||||
mentionsCypher := `
|
||||
MATCH (reply:Event {id: $replyId})-[:MENTIONS]->(author:Author {pubkey: $authorPubkey})
|
||||
RETURN author.pubkey AS pubkey
|
||||
`
|
||||
mentionsParams := map[string]any{
|
||||
"replyId": hex.Enc(replyEvent.ID[:]),
|
||||
"authorPubkey": hex.Enc(alice.Pub()),
|
||||
}
|
||||
|
||||
mentionsResult, err := db.ExecuteRead(ctx, mentionsCypher, mentionsParams)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to query MENTIONS relationship: %v", err)
|
||||
}
|
||||
|
||||
if !mentionsResult.Next(ctx) {
|
||||
t.Error("Expected MENTIONS relationship for p-tag")
|
||||
} else {
|
||||
t.Logf("✓ MENTIONS relationship verified")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSaveEvent_ETagMissingReference tests that e-tags to non-existent events
|
||||
// don't create broken relationships (OPTIONAL MATCH handles this gracefully).
|
||||
func TestSaveEvent_ETagMissingReference(t *testing.T) {
|
||||
neo4jURI := os.Getenv("ORLY_NEO4J_URI")
|
||||
if neo4jURI == "" {
|
||||
t.Skip("Skipping Neo4j test: ORLY_NEO4J_URI not set")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
tempDir := t.TempDir()
|
||||
db, err := New(ctx, cancel, tempDir, "debug")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
<-db.Ready()
|
||||
|
||||
if err := db.Wipe(); err != nil {
|
||||
t.Fatalf("Failed to wipe database: %v", err)
|
||||
}
|
||||
|
||||
signer, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := signer.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
// Create an event that references a non-existent event
|
||||
nonExistentEventID := "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
|
||||
|
||||
ev := event.New()
|
||||
ev.Pubkey = signer.Pub()
|
||||
ev.CreatedAt = timestamp.Now().V
|
||||
ev.Kind = 1
|
||||
ev.Content = []byte("Reply to ghost event")
|
||||
ev.Tags = tag.NewS(
|
||||
tag.NewFromAny("e", nonExistentEventID, "", "reply"),
|
||||
)
|
||||
|
||||
if err := ev.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign event: %v", err)
|
||||
}
|
||||
|
||||
// Save should succeed (OPTIONAL MATCH handles missing reference)
|
||||
exists, err := db.SaveEvent(ctx, ev)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to save event with missing reference: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("Event should not exist yet")
|
||||
}
|
||||
|
||||
// Verify event was saved
|
||||
checkCypher := "MATCH (e:Event {id: $id}) RETURN e.id AS id"
|
||||
checkParams := map[string]any{"id": hex.Enc(ev.ID[:])}
|
||||
|
||||
result, err := db.ExecuteRead(ctx, checkCypher, checkParams)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to check event: %v", err)
|
||||
}
|
||||
|
||||
if !result.Next(ctx) {
|
||||
t.Error("Event should have been saved despite missing reference")
|
||||
}
|
||||
|
||||
// Verify no REFERENCES relationship was created (as the target doesn't exist)
|
||||
refCypher := `
|
||||
MATCH (e:Event {id: $eventId})-[:REFERENCES]->(ref:Event)
|
||||
RETURN count(ref) AS refCount
|
||||
`
|
||||
refParams := map[string]any{"eventId": hex.Enc(ev.ID[:])}
|
||||
|
||||
refResult, err := db.ExecuteRead(ctx, refCypher, refParams)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to check references: %v", err)
|
||||
}
|
||||
|
||||
if refResult.Next(ctx) {
|
||||
count := refResult.Record().Values[0].(int64)
|
||||
if count > 0 {
|
||||
t.Errorf("Expected no REFERENCES relationship for non-existent event, got %d", count)
|
||||
} else {
|
||||
t.Logf("✓ Correctly handled missing reference (no relationship created)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSaveEvent_MultipleETags tests events with multiple e-tags.
|
||||
func TestSaveEvent_MultipleETags(t *testing.T) {
|
||||
neo4jURI := os.Getenv("ORLY_NEO4J_URI")
|
||||
if neo4jURI == "" {
|
||||
t.Skip("Skipping Neo4j test: ORLY_NEO4J_URI not set")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
tempDir := t.TempDir()
|
||||
db, err := New(ctx, cancel, tempDir, "debug")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
<-db.Ready()
|
||||
|
||||
if err := db.Wipe(); err != nil {
|
||||
t.Fatalf("Failed to wipe database: %v", err)
|
||||
}
|
||||
|
||||
signer, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := signer.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
// Create three events first
|
||||
var eventIDs []string
|
||||
for i := 0; i < 3; i++ {
|
||||
ev := event.New()
|
||||
ev.Pubkey = signer.Pub()
|
||||
ev.CreatedAt = timestamp.Now().V + int64(i)
|
||||
ev.Kind = 1
|
||||
ev.Content = []byte(fmt.Sprintf("Event %d", i))
|
||||
|
||||
if err := ev.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign event %d: %v", i, err)
|
||||
}
|
||||
|
||||
if _, err := db.SaveEvent(ctx, ev); err != nil {
|
||||
t.Fatalf("Failed to save event %d: %v", i, err)
|
||||
}
|
||||
|
||||
eventIDs = append(eventIDs, hex.Enc(ev.ID[:]))
|
||||
}
|
||||
|
||||
// Create an event that references all three
|
||||
replyEvent := event.New()
|
||||
replyEvent.Pubkey = signer.Pub()
|
||||
replyEvent.CreatedAt = timestamp.Now().V + 10
|
||||
replyEvent.Kind = 1
|
||||
replyEvent.Content = []byte("This references multiple events")
|
||||
replyEvent.Tags = tag.NewS(
|
||||
tag.NewFromAny("e", eventIDs[0], "", "root"),
|
||||
tag.NewFromAny("e", eventIDs[1], "", "reply"),
|
||||
tag.NewFromAny("e", eventIDs[2], "", "mention"),
|
||||
)
|
||||
|
||||
if err := replyEvent.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign reply event: %v", err)
|
||||
}
|
||||
|
||||
// Save reply event - tests multiple OPTIONAL MATCH statements after WITH
|
||||
exists, err := db.SaveEvent(ctx, replyEvent)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to save multi-reference event: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("Reply event should not exist yet")
|
||||
}
|
||||
|
||||
// Verify all REFERENCES relationships were created
|
||||
cypher := `
|
||||
MATCH (reply:Event {id: $replyId})-[:REFERENCES]->(ref:Event)
|
||||
RETURN ref.id AS refId
|
||||
`
|
||||
params := map[string]any{"replyId": hex.Enc(replyEvent.ID[:])}
|
||||
|
||||
result, err := db.ExecuteRead(ctx, cypher, params)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to query REFERENCES relationships: %v", err)
|
||||
}
|
||||
|
||||
referencedIDs := make(map[string]bool)
|
||||
for result.Next(ctx) {
|
||||
refID := result.Record().Values[0].(string)
|
||||
referencedIDs[refID] = true
|
||||
}
|
||||
|
||||
if len(referencedIDs) != 3 {
|
||||
t.Errorf("Expected 3 REFERENCES relationships, got %d", len(referencedIDs))
|
||||
}
|
||||
|
||||
for i, id := range eventIDs {
|
||||
if !referencedIDs[id] {
|
||||
t.Errorf("Missing REFERENCES relationship to event %d (%s)", i, id[:8])
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("✓ All %d REFERENCES relationships created successfully", len(referencedIDs))
|
||||
}
|
||||
|
||||
// TestBuildEventCreationCypher_CypherSyntaxValidation validates the generated Cypher
|
||||
// is syntactically correct for all edge cases.
|
||||
func TestBuildEventCreationCypher_CypherSyntaxValidation(t *testing.T) {
|
||||
neo4jURI := os.Getenv("ORLY_NEO4J_URI")
|
||||
if neo4jURI == "" {
|
||||
t.Skip("Skipping Neo4j test: ORLY_NEO4J_URI not set")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
tempDir := t.TempDir()
|
||||
db, err := New(ctx, cancel, tempDir, "debug")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
<-db.Ready()
|
||||
|
||||
signer, err := p8k.New()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create signer: %v", err)
|
||||
}
|
||||
if err := signer.Generate(); err != nil {
|
||||
t.Fatalf("Failed to generate keypair: %v", err)
|
||||
}
|
||||
|
||||
// Test many e-tags to ensure WITH clause is added only once
|
||||
manyETags := tag.NewS()
|
||||
for i := 0; i < 10; i++ {
|
||||
manyETags.Append(tag.NewFromAny("e", fmt.Sprintf("%064x", i)))
|
||||
}
|
||||
|
||||
ev := event.New()
|
||||
ev.Pubkey = signer.Pub()
|
||||
ev.CreatedAt = timestamp.Now().V
|
||||
ev.Kind = 1
|
||||
ev.Content = []byte("Event with many e-tags")
|
||||
ev.Tags = manyETags
|
||||
|
||||
if err := ev.Sign(signer); err != nil {
|
||||
t.Fatalf("Failed to sign event: %v", err)
|
||||
}
|
||||
|
||||
cypher, _ := db.buildEventCreationCypher(ev, 1)
|
||||
|
||||
// Count occurrences of WITH clause - should be exactly 1
|
||||
withCount := strings.Count(cypher, "WITH e, a")
|
||||
if withCount != 1 {
|
||||
t.Errorf("Expected exactly 1 WITH clause, found %d\nCypher:\n%s", withCount, cypher)
|
||||
}
|
||||
|
||||
// Count OPTIONAL MATCH statements - should equal number of e-tags
|
||||
optionalMatchCount := strings.Count(cypher, "OPTIONAL MATCH")
|
||||
if optionalMatchCount != 10 {
|
||||
t.Errorf("Expected 10 OPTIONAL MATCH statements, found %d", optionalMatchCount)
|
||||
}
|
||||
|
||||
t.Logf("✓ WITH clause correctly added once, followed by %d OPTIONAL MATCH statements", optionalMatchCount)
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
v0.31.8
|
||||
v0.31.10
|
||||
Reference in New Issue
Block a user