diff --git a/go.mod b/go.mod index a36bb653..0f05b248 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/docker/docker v20.10.22+incompatible github.com/gookit/color v1.5.2 github.com/hashicorp/golang-lru/v2 v2.0.2 + github.com/indra-labs/go-ds-badger3 v0.3.0 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-ds-badger v0.3.0 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 @@ -46,7 +46,6 @@ require ( ) require ( - github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/Microsoft/hcsshim v0.9.6 // indirect @@ -75,7 +74,6 @@ require ( github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect github.com/decred/dcrd/lru v1.0.0 // indirect - github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect diff --git a/go.sum b/go.sum index 3b435c34..f0b1f708 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,6 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -369,11 +367,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3 github.com/decred/dcrd/lru v1.0.0 h1:Kbsb1SFDsIlaupWPwsPp+dkxiBY1frcS07PCPgotKz8= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= -github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -691,6 +686,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/indra-labs/go-ds-badger3 v0.3.0 h1:p40sV84SmiTAtz9qv4PACKQ22RO7JZMUcd+ZZtQ9IaI= +github.com/indra-labs/go-ds-badger3 v0.3.0/go.mod h1:JqTSNITguQWrixcPO3s4xhNFA4jtYjpocBx+6jSIhHk= github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -700,8 +697,6 @@ github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0M github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= -github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -709,7 +704,6 @@ github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2PO github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= -github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= @@ -1426,7 +1420,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= diff --git a/pkg/cert/ad.go b/pkg/cert/ad.go index ac5009cf..bc9a3496 100644 --- a/pkg/cert/ad.go +++ b/pkg/cert/ad.go @@ -4,12 +4,7 @@ package cert import ( "github.com/indra-labs/indra/pkg/codec" "github.com/indra-labs/indra/pkg/crypto" - log2 "github.com/indra-labs/indra/pkg/proc/log" -) - -var ( - log = log2.GetLogger() - fails = log.E.Chk + "github.com/libp2p/go-libp2p/core/peer" ) // Act is an interface for the signed messages stored in the PeerStore of the @@ -19,22 +14,11 @@ var ( // other types of contract documents. In the language of Law an Act is the // prototype of a declaration or claim, a land title is an example of a type of // Act. -// -// In Indra, this is a spam-resistant message type that invites validation and -// indirectly spammy abuse of this message type in the gossip network will cause -// banning and the eviction of the record. -// -// Hidden services introduction advertisement is an example of the hidden -// service attesting to the provision of the referral messages found in the -// package: pkg/codec/onion/hidden/services - These are expected to become far -// more numerous than peer advertisements as they effectively designate a -// listening server. These can be spam-controlled by having peers poke at the -// service and dropping non-working intros and thus potentially leading to the -// hidden service intro being evicted from the collective peerstore. -// -// todo: hidden service sessions... type Act interface { codec.Codec Sign(key *crypto.Prv) (e error) Validate() bool + PubKey() (pubKey *crypto.Pub) + GetID() (id peer.ID, e error) + Expired() (is bool) } diff --git a/pkg/codec/ad/addresses/addresses.go b/pkg/codec/ad/addresses/addresses.go index 4fede5b4..bfaa8464 100644 --- a/pkg/codec/ad/addresses/addresses.go +++ b/pkg/codec/ad/addresses/addresses.go @@ -10,6 +10,7 @@ import ( log2 "github.com/indra-labs/indra/pkg/proc/log" "github.com/indra-labs/indra/pkg/util/slice" "github.com/indra-labs/indra/pkg/util/splice" + "github.com/libp2p/go-libp2p/core/peer" "net/netip" "time" ) @@ -62,6 +63,14 @@ func New(id nonce.ID, key *crypto.Prv, addrs []*netip.AddrPort, return } +func (x *Ad) PubKey() (key *crypto.Pub) { return x.Key } +func (x *Ad) Fingerprint() (pf string) { return x.Key.Fingerprint() } +func (x *Ad) Expired() (is bool) { return x.Expiry.Before(time.Now()) } + +func (x *Ad) GetID() (id peer.ID, e error) { + return peer.IDFromPublicKey(x.Key) +} + // Decode a splice.Splice's next bytes into an Ad. func (x *Ad) Decode(s *splice.Splice) (e error) { var i, count uint16 @@ -94,7 +103,8 @@ func (x *Ad) Unwrap() interface{} { return nil } // Len returns the length of bytes required to encode the Ad, based on the number // of Addresses inside it. func (x *Ad) Len() int { - return ad.Len + len(x.Addresses)*(1+Len) + slice.Uint16Len + l := ad.Len + len(x.Addresses)*(1+splice.AddrLen) + slice.Uint16Len + return l } // Magic bytes that identify this message diff --git a/pkg/codec/ad/intro/intro.go b/pkg/codec/ad/intro/intro.go index e4b5a834..ddc20606 100644 --- a/pkg/codec/ad/intro/intro.go +++ b/pkg/codec/ad/intro/intro.go @@ -6,6 +6,7 @@ import ( "github.com/indra-labs/indra/pkg/codec/ad" "github.com/indra-labs/indra/pkg/codec/reg" log2 "github.com/indra-labs/indra/pkg/proc/log" + "github.com/libp2p/go-libp2p/core/peer" "time" "github.com/indra-labs/indra/pkg/crypto" @@ -45,6 +46,14 @@ type Ad struct { var _ codec.Codec = &Ad{} +func (x *Ad) PubKey() (key *crypto.Pub) { return x.Key } +func (x *Ad) Fingerprint() (pf string) { return x.Key.Fingerprint() } +func (x *Ad) Expired() (is bool) { return x.Expiry.Before(time.Now()) } + +func (x *Ad) GetID() (id peer.ID, e error) { + return peer.IDFromPublicKey(x.Key) +} + // Decode an Ad out of the next bytes of a splice.Splice. func (x *Ad) Decode(s *splice.Splice) (e error) { if e = magic.TooShort(s.Remaining(), Len-magic.Len, diff --git a/pkg/codec/ad/load/load.go b/pkg/codec/ad/load/load.go index f37a6f57..9532182f 100644 --- a/pkg/codec/ad/load/load.go +++ b/pkg/codec/ad/load/load.go @@ -9,6 +9,7 @@ import ( "github.com/indra-labs/indra/pkg/crypto/nonce" log2 "github.com/indra-labs/indra/pkg/proc/log" "github.com/indra-labs/indra/pkg/util/splice" + "github.com/libp2p/go-libp2p/core/peer" "time" ) @@ -34,6 +35,14 @@ type Ad struct { var _ codec.Codec = &Ad{} +func (x *Ad) PubKey() (key *crypto.Pub) { return x.Key } +func (x *Ad) Fingerprint() (pf string) { return x.Key.Fingerprint() } +func (x *Ad) Expired() (is bool) { return x.Expiry.Before(time.Now()) } + +func (x *Ad) GetID() (id peer.ID, e error) { + return peer.IDFromPublicKey(x.Key) +} + // New creates a new Ad. func New(id nonce.ID, key *crypto.Prv, load byte, expiry time.Time) (loAd *Ad) { diff --git a/pkg/codec/ad/peer/peer.go b/pkg/codec/ad/peer/peer.go index 1ec32ca5..8145debb 100644 --- a/pkg/codec/ad/peer/peer.go +++ b/pkg/codec/ad/peer/peer.go @@ -12,6 +12,7 @@ import ( log2 "github.com/indra-labs/indra/pkg/proc/log" "github.com/indra-labs/indra/pkg/util/slice" "github.com/indra-labs/indra/pkg/util/splice" + "github.com/libp2p/go-libp2p/core/peer" "time" ) @@ -38,6 +39,14 @@ type Ad struct { var _ codec.Codec = &Ad{} +func (x *Ad) PubKey() (key *crypto.Pub) { return x.Key } +func (x *Ad) Fingerprint() (pf string) { return x.Key.Fingerprint() } +func (x *Ad) Expired() (is bool) { return x.Expiry.Before(time.Now()) } + +func (x *Ad) GetID() (id peer.ID, e error) { + return peer.IDFromPublicKey(x.Key) +} + // New creates a new Ad and signs it with the provided private key. func New(id nonce.ID, key *crypto.Prv, relayRate uint32, expiry time.Time) (peerAd *Ad) { diff --git a/pkg/codec/ad/services/services.go b/pkg/codec/ad/services/services.go index c1d26221..3def4216 100644 --- a/pkg/codec/ad/services/services.go +++ b/pkg/codec/ad/services/services.go @@ -10,6 +10,7 @@ import ( log2 "github.com/indra-labs/indra/pkg/proc/log" "github.com/indra-labs/indra/pkg/util/slice" "github.com/indra-labs/indra/pkg/util/splice" + "github.com/libp2p/go-libp2p/core/peer" "time" ) @@ -67,6 +68,14 @@ func New(id nonce.ID, key *crypto.Prv, services []Service, return } +func (x *Ad) PubKey() (key *crypto.Pub) { return x.Key } +func (x *Ad) Fingerprint() (pf string) { return x.Key.Fingerprint() } +func (x *Ad) Expired() (is bool) { return x.Expiry.Before(time.Now()) } + +func (x *Ad) GetID() (id peer.ID, e error) { + return peer.IDFromPublicKey(x.Key) +} + // Decode an Ad out of the next bytes of a splice.Splice. func (x *Ad) Decode(s *splice.Splice) (e error) { var i, count uint16 diff --git a/pkg/codec/onion/hidden/services/hiddenservice.go b/pkg/codec/onion/hidden/services/hiddenservice.go index 9dc2e666..c3baaf6d 100644 --- a/pkg/codec/onion/hidden/services/hiddenservice.go +++ b/pkg/codec/onion/hidden/services/hiddenservice.go @@ -90,9 +90,11 @@ func (x *HiddenService) Decode(s *splice.Splice) (e error) { // Encode a HiddenService into a the next bytes of a Splice. func (x *HiddenService) Encode(s *splice.Splice) (e error) { log.T.S("encoding", reflect.TypeOf(x), - x.Intro.ID, x.Intro.Key, x.Intro.Introducer, x.Ciphers, x.Nonces, x.RoutingHeaderBytes, + x.Intro.GetID, x.Intro.Key, x.Intro.Introducer, x.Ciphers, x.Nonces, x.RoutingHeaderBytes, ) - x.Intro.Encode(s.Magic(Magic)) + if e = x.Intro.Encode(s.Magic(Magic)); fails(e) { + return + } return x.Onion.Encode(s.Ciphers(x.Ciphers).Nonces(x.Nonces)) } diff --git a/pkg/codec/ont/interfaces.go b/pkg/codec/ont/interfaces.go index b58856e4..9fdbe73a 100644 --- a/pkg/codec/ont/interfaces.go +++ b/pkg/codec/ont/interfaces.go @@ -84,7 +84,7 @@ type Onion interface { last bool) (skip bool, sd *sessions.Data) } -// Encode is the generic encoder for an onion, all onions can be encoded with it. +// Encode is the generic encoder for a Codec, all can be encoded with it. func Encode(d Codec) (s *splice.Splice) { s = splice.New(d.Len()) fails(d.Encode(s)) diff --git a/pkg/codec/reg/registry.go b/pkg/codec/reg/registry.go index ab9b29af..feef6e19 100644 --- a/pkg/codec/reg/registry.go +++ b/pkg/codec/reg/registry.go @@ -11,7 +11,6 @@ import ( "github.com/indra-labs/indra/pkg/codec" log2 "github.com/indra-labs/indra/pkg/proc/log" "github.com/indra-labs/indra/pkg/util/splice" - "reflect" "sync" ) @@ -58,10 +57,10 @@ func Recognise(s *splice.Splice) (cdc codec.Codec) { spew.Sdump(s.GetUntil(s.GetCursor()).ToBytes()), spew.Sdump(s.GetFrom(s.GetCursor()).ToBytes()), ) - } else { - log.T.F("recognised magic %s for type %v", - color.Red.Sprint(magic), - color.Green.Sprint(reflect.TypeOf(cdc))) + // } else { + // log.T.F("recognised magic %s for type %v", + // color.Red.Sprint(magic), + // color.Green.Sprint(reflect.TypeOf(cdc))) } return } diff --git a/pkg/crypto/crypto.go b/pkg/crypto/crypto.go index 779a36f2..671bd8dd 100644 --- a/pkg/crypto/crypto.go +++ b/pkg/crypto/crypto.go @@ -248,11 +248,11 @@ func (p *Prv) Zero() { (*secp256k1.PrivateKey)(p).Zero() } // Pub is a public key. type Pub secp256k1.PublicKey -// Fingerprint generates a compact and distinctive Based32 fingeprint to easily +// Fingerprint generates a compact and distinctive Based32 fingerprint to easily // distinguish between many peers at a glance. // -// It is generated with a SHA256 hash of the identity key snipped to yield an 8 -// character string. The truncation is done after the string encoding. +// It is generated with a SHA256 hash of the identity key snipped at 40 bits (5 +// bytes) to yield an 8 character Based32 string fingerprint. func (k *Pub) Fingerprint() (fp string) { kk := k.ToBytes() b := sha256.Single(kk[:]) diff --git a/pkg/engine/ads/ads.go b/pkg/engine/ads/ads.go index d5974e87..ae8f1f68 100644 --- a/pkg/engine/ads/ads.go +++ b/pkg/engine/ads/ads.go @@ -3,6 +3,7 @@ package ads import ( "errors" + "github.com/indra-labs/indra/pkg/cert" "github.com/indra-labs/indra/pkg/codec/ad" "github.com/indra-labs/indra/pkg/codec/ad/addresses" "github.com/indra-labs/indra/pkg/codec/ad/load" @@ -42,6 +43,10 @@ type NodeAds struct { Load *load.Ad } +func (na *NodeAds) GetAsCerts() (ads []cert.Act) { + return []cert.Act{na.Address, na.Load, na.Peer, na.Services} +} + // GetMultiaddrs returns a node's listener addresses. func GetMultiaddrs(n *node.Node) (ma []multiaddr.Multiaddr, e error) { for i := range n.Addresses { diff --git a/pkg/engine/dispatcher/dispatcher_test.go b/pkg/engine/dispatcher/dispatcher_test.go index d2163144..cd0b2943 100644 --- a/pkg/engine/dispatcher/dispatcher_test.go +++ b/pkg/engine/dispatcher/dispatcher_test.go @@ -2,8 +2,10 @@ package dispatcher import ( "context" + "crypto/rand" "github.com/indra-labs/indra/pkg/codec/onion/cores/confirmation" "github.com/indra-labs/indra/pkg/codec/onion/cores/response" + "github.com/indra-labs/indra/pkg/crypto/sha256" "os" "testing" "time" @@ -42,7 +44,9 @@ func TestDispatcher(t *testing.T) { if err != nil { t.FailNow() } - store, closer := transport.BadgerStore(dataPath) + secret := sha256.New() + rand.Read(secret[:]) + store, closer := transport.BadgerStore(dataPath, secret[:]) if store == nil { t.Fatal("could not open database") } @@ -56,7 +60,9 @@ func TestDispatcher(t *testing.T) { if err != nil { t.FailNow() } - store, closer = transport.BadgerStore(dataPath) + secret = sha256.New() + rand.Read(secret[:]) + store, closer = transport.BadgerStore(dataPath, secret[:]) if store == nil { t.Fatal("could not open database") } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 63846a85..34ace187 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -2,7 +2,6 @@ package engine import ( "context" - "github.com/indra-labs/indra/pkg/cert" "github.com/indra-labs/indra/pkg/codec/ont" "github.com/indra-labs/indra/pkg/codec/reg" "github.com/indra-labs/indra/pkg/crypto" @@ -106,8 +105,7 @@ out: cancel() return } - na := ng.NodeAds - a := []cert.Act{na.Address, na.Load, na.Peer, na.Services} + a := ng.NodeAds.GetAsCerts() for i := range a { if e = a[i].Sign(ng.Mgr().GetLocalNodeIdentityPrv()); fails(e) { cancel() @@ -131,7 +129,7 @@ func (ng *Engine) Shutdown() { if ng.ShuttingDown.Load() { return } - log.T.Ln("shutting down", ng.Mgr().GetLocalNodeAddress().String()) + log.T.Ln(ng.LogEntry("shutting down")) ng.ShuttingDown.Store(true) ng.Cleanup() ng.cancel() diff --git a/pkg/engine/mock.go b/pkg/engine/mock.go index 4f589f7a..946040cc 100644 --- a/pkg/engine/mock.go +++ b/pkg/engine/mock.go @@ -2,9 +2,11 @@ package engine import ( "context" + "crypto/rand" "errors" "github.com/indra-labs/indra/pkg/crypto" "github.com/indra-labs/indra/pkg/crypto/nonce" + "github.com/indra-labs/indra/pkg/crypto/sha256" "github.com/indra-labs/indra/pkg/engine/node" "github.com/indra-labs/indra/pkg/engine/sessions" "github.com/indra-labs/indra/pkg/engine/tpt" @@ -64,7 +66,9 @@ func createNMockCircuits(inclSessions bool, nCircuits int, if k, e = crypto.GenerateKeys(); fails(e) { return } - store, closer := transport.BadgerStore(dataPath) + secret := sha256.New() + rand.Read(secret[:]) + store, closer := transport.BadgerStore(dataPath, secret[:]) if store == nil { return nil, errors.New("could not open database") } diff --git a/pkg/engine/mockengine.go b/pkg/engine/mockengine.go index 89923001..f01eea9a 100644 --- a/pkg/engine/mockengine.go +++ b/pkg/engine/mockengine.go @@ -2,7 +2,10 @@ package engine import ( "context" + "crypto/rand" + badger "github.com/indra-labs/go-ds-badger3" "github.com/indra-labs/indra/pkg/crypto" + "github.com/indra-labs/indra/pkg/crypto/sha256" "github.com/indra-labs/indra/pkg/engine/node" "github.com/indra-labs/indra/pkg/engine/transport" "github.com/indra-labs/indra/pkg/util/multi" @@ -12,26 +15,31 @@ import ( ) // CreateMockEngine creates an indra Engine with a random localhost listener. -func CreateMockEngine(seed []string, dataPath string, - ctx context.Context, cancel context.CancelFunc) (ng *Engine) { +func CreateMockEngine(seed []string, dataPath string, ctx context.Context, cancel context.CancelFunc) (ng *Engine, store *badger.Datastore, closer func()) { - var e error + var ( + e error + keys []*crypto.Keys + k *crypto.Keys + ) defer func(f *error) { if *f != nil { fails(os.RemoveAll(dataPath)) } }(&e) - var keys []*crypto.Keys - var k *crypto.Keys if k, e = crypto.GenerateKeys(); fails(e) { return } keys = append(keys, k) - store, closer := transport.BadgerStore(dataPath) + + secret := sha256.New() + rand.Read(secret[:]) + store, closer = transport.BadgerStore(dataPath, secret[:]) if store == nil { log.E.Ln("could not open database") - return nil + return nil, store, closer } + var l *transport.Listener if l, e = transport.NewListener(seed, []string{transport.LocalhostZeroIPv4TCP, @@ -42,15 +50,19 @@ func CreateMockEngine(seed []string, dataPath string, if l == nil { panic("maybe you have no network device?") } + sa := transport.GetHostFirstMultiaddr(l.Host) - var ap netip.AddrPort + var ma multiaddr.Multiaddr if ma, e = multiaddr.NewMultiaddr(sa); fails(e) { return } + + var ap netip.AddrPort if ap, e = multi.AddrToAddrPort(ma); fails(e) { return } + var nod *node.Node if nod, _ = node.NewNode([]*netip.AddrPort{&ap}, k, nil, 50000); fails(e) { return @@ -66,9 +78,9 @@ func CreateMockEngine(seed []string, dataPath string, } func CreateAndStartMockEngines(n int, ctx context.Context, - cancel context.CancelFunc) (engines []*Engine, cleanup func(), e error) { + cancel context.CancelFunc) (engines []*Engine, closer func(), e error) { - cleanup = func() {} + closer = func() {} var seed []string dataPath := make([]string, n) for i := 0; i < n; i++ { @@ -77,7 +89,7 @@ func CreateAndStartMockEngines(n int, ctx context.Context, return } var eng *Engine - if eng = CreateMockEngine(seed, dataPath[i], ctx, cancel); fails(e) { + if eng, _, _ = CreateMockEngine(seed, dataPath[i], ctx, cancel); fails(e) { return } engines = append(engines, eng) @@ -86,7 +98,7 @@ func CreateAndStartMockEngines(n int, ctx context.Context, } go eng.Start() } - cleanup = func() { + closer = func() { for i := range engines { if engines[i] != nil { engines[i].Shutdown() diff --git a/pkg/engine/peerstore.go b/pkg/engine/peerstore.go index 01aae7e3..b6b3df62 100644 --- a/pkg/engine/peerstore.go +++ b/pkg/engine/peerstore.go @@ -4,8 +4,9 @@ import ( "context" "errors" "fmt" + "github.com/dgraph-io/badger/v3" "github.com/indra-labs/indra/pkg/cert" - "github.com/indra-labs/indra/pkg/codec/ad/load" + magic2 "github.com/indra-labs/indra/pkg/engine/magic" "github.com/indra-labs/indra/pkg/util/slice" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" @@ -13,9 +14,6 @@ import ( "time" "github.com/indra-labs/indra/pkg/codec/ad/addresses" - "github.com/indra-labs/indra/pkg/codec/ad/intro" - peer2 "github.com/indra-labs/indra/pkg/codec/ad/peer" - "github.com/indra-labs/indra/pkg/codec/ad/services" "github.com/indra-labs/indra/pkg/codec/reg" "github.com/indra-labs/indra/pkg/util/splice" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -52,11 +50,13 @@ func (ng *Engine) SendAd(a slice.Bytes) (e error) { // SendAds dispatches all ads in NodeAds. Primarily called at startup. func (ng *Engine) SendAds() (e error) { - na := ng.NodeAds - ads := []cert.Act{na.Address, na.Load, na.Peer, na.Services} - for i := range ads { - s := splice.New(ads[i].Len()) - ads[i].Encode(s) + ads := ng.NodeAds.GetAsCerts() + for _, v := range ads { + + s := splice.New(v.Len()) + if fails(v.Encode(s)) { + return + } if e = ng.topic.Publish(ng.ctx, s.GetAll()); fails(e) { return } @@ -112,67 +112,31 @@ func (ng *Engine) LogEntry(s string) (entry string) { } func (ng *Engine) gossip(tick *time.Ticker) { - now := time.Now() first := true out: for { if first { first = false // Send out all ads because we are starting up. - ng.SendAds() + fails(ng.SendAds()) // As all ads are sent we can return to the head of the loop. continue } // Check for already generated NodeAds, and make them first time if // needed. - na := ng.NodeAds log.D.Ln(ng.LogEntry("gossip tick")) - switch { - case na.Address == nil: - log.D.Ln(ng.LogEntry("updating peer address")) - - fallthrough - - case na.Load == nil: - log.D.Ln(ng.LogEntry("updating peer load")) - - fallthrough - - case na.Peer == nil: - log.D.Ln(ng.LogEntry("updating peer ad")) - - fallthrough - - case na.Services == nil && - // But only if we have any services: - len(ng.Mgr().GetLocalNode().Services) > 0: - log.D.Ln(ng.LogEntry("updating services")) - - fallthrough - // Next, check each entry has not expired: - - case na.Address.Expiry.Before(now): - log.D.Ln(ng.LogEntry("updating expired peer address")) - - fallthrough - - case na.Load.Expiry.Before(now): - log.D.Ln(ng.LogEntry("updating expired load ad")) - - fallthrough - - case na.Peer.Expiry.Before(now): - log.D.Ln(ng.LogEntry("updating peer ad")) - - fallthrough - - case na.Services.Expiry.Before(now): - log.D.Ln(ng.LogEntry("updating peer services")) - + var e error + ads := ng.NodeAds.GetAsCerts() + for i := range ads { + if !ads[i].Expired() { + continue + } + s := splice.New(ads[i].Len()) + ads[i].Encode(s) + if e = ng.topic.Publish(ng.ctx, s.GetAll()); fails(e) { + return + } } - // Then, lastly, check if the ad content has changed due to - // reconfiguration or other reasons such as a more substantial amount of - // load or drop in load, or changed IP addresses. // After all that is done, check if we are shutting down, if so exit. select { case <-ng.ctx.Done(): @@ -206,129 +170,39 @@ func (ng *Engine) HandleAd(p *pubsub.Message) (e error) { return } var ok bool - switch c.(type) { - case *addresses.Ad: - var addr *addresses.Ad - if addr, ok = c.(*addresses.Ad); !ok { - return fmt.Errorf(ErrWrongTypeDecode, - addresses.Magic, reflect.TypeOf(c).String()) - } else if !addr.Validate() { - return errors.New("addr ad failed validation") - } - // No need to store our own (why does pubsub do this?) - if addr.Key.Fingerprint() == ng.Listener.Pub.Fingerprint() { - break - } - log.D.Ln(ng.LogEntry("received"), reflect.TypeOf(c), - "from gossip network for node", addr.Key.Fingerprint()) - // If we got to here now we can add to the PeerStore. - var id peer.ID - if id, e = peer.IDFromPublicKey(addr.Key); fails(e) { - return - } - if id != ng.Listener.Host.ID() { - if e = ng.Listener.Host. - Peerstore().Put(id, addresses.Magic, s.GetAll().ToBytes()); fails(e) { - return - } - } - case *intro.Ad: - var intr *intro.Ad - if intr, ok = c.(*intro.Ad); !ok { - return fmt.Errorf(ErrWrongTypeDecode, - intro.Magic, reflect.TypeOf(c).String()) - } else if !intr.Validate() { - return errors.New("intro ad failed validation") - } - // We don't need to store introductions we are hosting again. - if intr.Introducer.Fingerprint() == ng.Listener.Pub.Fingerprint() { - break - } - log.D.Ln(ng.LogEntry("received"), reflect.TypeOf(c), - "from gossip network for node", intr.Key.Fingerprint()) - // If we got to here now we can add to the PeerStore. - var id peer.ID - if id, e = peer.IDFromPublicKey(intr.Key); fails(e) { - return - } - if e = ng.Listener.Host. - Peerstore().Put(id, intro.Magic, s.GetAll().ToBytes()); fails(e) { - return - } - case *load.Ad: - var lod *load.Ad - if lod, ok = c.(*load.Ad); !ok { - return fmt.Errorf(ErrWrongTypeDecode, - addresses.Magic, reflect.TypeOf(c).String()) - } else if !lod.Validate() { - return errors.New("load ad failed validation") - } - if lod.Key.Fingerprint() == ng.Listener.Pub.Fingerprint() { - break - } - log.D.Ln(ng.LogEntry("received"), reflect.TypeOf(c), - "from gossip network for node", lod.Key.Fingerprint()) - // If we got to here now we can add to the PeerStore. - var id peer.ID - if id, e = peer.IDFromPublicKey(lod.Key); fails(e) { - return - } - log.T.Ln(ng.LogEntry("storing ad")) - if e = ng.Listener.Host. - Peerstore().Put(id, load.Magic, s.GetAll().ToBytes()); fails(e) { - return - } - case *peer2.Ad: - var pa *peer2.Ad - if pa, ok = c.(*peer2.Ad); !ok { - return fmt.Errorf(ErrWrongTypeDecode, - peer2.Magic, reflect.TypeOf(c).String()) - } else if !pa.Validate() { - return errors.New("peer ad failed validation") - } - if pa.Key.Fingerprint() == ng.Listener.Pub.Fingerprint() { - break - } - log.D.Ln(ng.LogEntry("received"), reflect.TypeOf(c), - "from gossip network for node", pa.Key.Fingerprint()) - // If we got to here now we can add to the PeerStore. - var id peer.ID - if id, e = peer.IDFromPublicKey(pa.Key); fails(e) { - return - } - if e = ng.Listener.Host. - Peerstore().Put(id, peer2.Magic, s.GetAll().ToBytes()); fails(e) { - return - } - case *services.Ad: - var sa *services.Ad - if sa, ok = c.(*services.Ad); !ok { - return fmt.Errorf(ErrWrongTypeDecode, - services.Magic, reflect.TypeOf(c).String()) - } else if !sa.Validate() { - return errors.New("services ad failed validation") - } - if sa.Key.Fingerprint() == ng.Listener.Pub.Fingerprint() { - break - } - log.D.Ln(ng.LogEntry("received"), reflect.TypeOf(c), - "from gossip network for node", sa.Key.Fingerprint()) - // If we got to here now we can add to the PeerStore. - var id peer.ID - if id, e = peer.IDFromPublicKey(sa.Key); fails(e) { - return - } - if e = ng.Listener.Host. - Peerstore().Put(id, services.Magic, s.GetAll().ToBytes()); fails(e) { - return - } + var a cert.Act + if a, ok = c.(cert.Act); !ok { + return fmt.Errorf(ErrWrongTypeDecode, + addresses.Magic, reflect.TypeOf(c).String()) + } + var id peer.ID + if id, e = a.GetID(); fails(e) { + return + } + if id == ng.Listener.Host.ID() { + log.T.Ln("ignoring own ad") + } + if !a.Validate() { + return errors.New(reflect.TypeOf(c).String() + + "ad failed validation") + } + magic := string(s.GetAll()[:magic2.Len]) + log.D.Ln(ng.LogEntry("received"), reflect.TypeOf(c), + "from gossip network for node", a.PubKey().Fingerprint()) + + // If we got to here now we can add to the PeerStore. + log.D.Ln("storing value with magic", magic, s.GetAll()[:magic2.Len]) + if e = ng.Listener.Host. + Peerstore().Put(id, magic, s.GetAll().ToBytes()); fails(e) { + return } return } // GetPeerRecord queries the peerstore for an ad from a given peer.ID and the ad -// type key. The ad type keys are the same as the Magic of each ad type, to be -// simple. +// type key. +// +// The ad type keys are the same as the Magic of each ad type, to be simple. func (ng *Engine) GetPeerRecord(id peer.ID, key string) (add cert.Act, e error) { var a interface{} if a, e = ng.Listener.Host.Peerstore().Get(id, key); fails(e) { @@ -358,19 +232,28 @@ func (ng *Engine) GetPeerRecord(id peer.ID, key string) (add cert.Act, e error) return } -func (ng *Engine) IteratePeerRecords(func()) { +// PeerstoreUpdate provides access to the badger.DB to walk the records with a +// closure, that can update the record. +func (ng *Engine) PeerstoreUpdate(fn func(txn *badger.Txn) error) (e error) { + return ng.Listener.Datastore.DB.Update(fn) +} +// PeerstoreView provides access to the badger.DB to walk the records with a +// closure, that can update the record. +func (ng *Engine) PeerstoreView(fn func(txn *badger.Txn) error) (e error) { + return ng.Listener.Datastore.DB.View(fn) } // ClearPeerRecord places an empty slice into a peer record by way of deleting it. // -// todo: these should be purged from the peerstore in a GC pass. Expiry and storage limits... +// todo: these should be purged from the peerstore in a GC pass. +// todo: Expiry and storage limits... +// todo: The PeerstoreUpdate function can delete records. (probably absent this why nobody uses it). func (ng *Engine) ClearPeerRecord(id peer.ID, key string) (e error) { if _, e = ng.Listener.Host.Peerstore().Get(id, key); fails(e) { return } if e = ng.Listener.Host.Peerstore().Put(id, key, []byte{}); fails(e) { - return } return } diff --git a/pkg/engine/peerstore_test.go b/pkg/engine/peerstore_test.go index bfe97bc4..0ffad66c 100644 --- a/pkg/engine/peerstore_test.go +++ b/pkg/engine/peerstore_test.go @@ -2,11 +2,12 @@ package engine import ( "context" + "github.com/dgraph-io/badger/v3" "github.com/indra-labs/indra" + log2 "github.com/indra-labs/indra/pkg/proc/log" + "strings" "testing" "time" - - log2 "github.com/indra-labs/indra/pkg/proc/log" ) func pauza() { @@ -94,29 +95,64 @@ func pauza() { func TestEngine_PeerStoreDiscovery(t *testing.T) { if indra.CI == "false" { - log2.SetLogLevel(log2.Trace) + // log2.SetLogLevel(log2.Trace) } const nTotal = 10 - var e error - var engines []*Engine - var cleanup func() + var ( + e error + engines []*Engine + cleanup func() + ) ctx, cancel := context.WithCancel(context.Background()) - if engines, cleanup, e = CreateAndStartMockEngines(nTotal, ctx, cancel); fails(e) { + if engines, cleanup, e = CreateAndStartMockEngines(nTotal, ctx, + cancel); fails(e) { + t.FailNow() } - time.Sleep(time.Second * 2) + time.Sleep(time.Second * 3) // Send them all again after a bit to make sure everyone gets them. for i := range engines { if e = engines[i].SendAds(); fails(e) { t.FailNow() } } - time.Sleep(time.Second * 2) - for i := range engines { - _ = i + time.Sleep(time.Second * 3) + if indra.CI == "false" { + log2.SetLogLevel(log2.Debug) + } + var ec int + entryCount := &ec + for _, v := range engines { // check that all peers now have nTotal-1 distinct peer ads (of all 4 // types) - + e = v.PeerstoreView(func(txn *badger.Txn) error { + defer txn.Discard() + opts := badger.DefaultIteratorOptions + it := txn.NewIterator(opts) + defer it.Close() + var val []byte + var adCount int + for it.Rewind(); it.Valid(); it.Next() { + k := string(it.Item().Key()) + if !strings.HasSuffix(k, "ad") { + continue + } + val, e = it.Item().ValueCopy(nil) + log.T.S(v.LogEntry("item "+k), val) + adCount++ + } + log.T.Ln("adCount", adCount) + if adCount == (nTotal-1)*4 { + *entryCount++ + } + return nil + }) + } + if *entryCount != nTotal { + t.Error("nodes did not gossip completely to each other, only", + *entryCount, "nodes ad sets counted, not the expected", + nTotal) + t.FailNow() } cleanup() pauza() diff --git a/pkg/engine/transport/common.go b/pkg/engine/transport/common.go index 95b5f421..104eb563 100644 --- a/pkg/engine/transport/common.go +++ b/pkg/engine/transport/common.go @@ -3,9 +3,10 @@ package transport import ( "fmt" "github.com/gookit/color" + badger "github.com/indra-labs/go-ds-badger3" "github.com/indra-labs/indra" log2 "github.com/indra-labs/indra/pkg/proc/log" - "github.com/ipfs/go-ds-badger" + "github.com/indra-labs/indra/pkg/util/options" "github.com/libp2p/go-libp2p/core/host" "github.com/multiformats/go-multiaddr" ) @@ -83,10 +84,32 @@ func GetHostOnlyMultiaddrs(ha host.Host) (addrs []string) { } // BadgerStore creates a new badger database backed persistence engine for keys -// and values used in the peer information database. -func BadgerStore(dataPath string) (store *badger.Datastore, closer func()) { +// and values used in the peer information database, basically keeps a +// collection of all received advertisments on the network keyed to their public +// key. +// +// This database includes a data structure that maps and records history related +// to: +// +// - keys of hidden connections - having several past versions can help avoid +// needing to find an introducer again. +// - keys of relay to relay and client to relay connections receiver public +// keys. +// +// This database must be encrypted as the data stored in it includes the +// libp2p.Host private key. +func BadgerStore(dataPath string, + cipher []byte) (store *badger.Datastore, + closer func()) { + + if cipher == nil { + panic("DANGER: database is being created without encryption") + } + log.T.Ln("dataPath", dataPath) - store, err := badger.NewDatastore(dataPath, nil) + o := options.Default(dataPath, cipher[:]) + opts := &badger.Options{Options: *o} + store, err := badger.NewDatastore(dataPath, opts) if fails(err) { return nil, func() {} } diff --git a/pkg/engine/transport/discovery.go b/pkg/engine/transport/discovery.go index 6156d018..eea62053 100644 --- a/pkg/engine/transport/discovery.go +++ b/pkg/engine/transport/discovery.go @@ -40,8 +40,8 @@ func NewDHT(ctx context.Context, host host.Host, } wg.Add(1) go func() { - if e := host.Connect(ctx, *peerinfo); fails(e) { - log.D.F("Error while connecting to node %q", peerinfo) + if e := host.Connect(ctx, *peerinfo); e != nil { + log.T.F("Error while connecting to node %q", peerinfo) wg.Done() return } @@ -68,7 +68,7 @@ func (l *Listener) Discover(ctx context.Context, h host.Host, dht *dht.IpfsDHT, if _, e = disco.Advertise(ctx, rendezvous[i].String()); e != nil { } } - if e = l.Tick(h, rendezvous, peers, disco, ctx); fails(e) { + if e = l.Tick(h, rendezvous, peers, disco, ctx); e != nil { } ticker := time.NewTicker(time.Second * 1) defer ticker.Stop() @@ -101,7 +101,7 @@ func (l *Listener) Tick(h host.Host, rendezvous []multiaddr.Multiaddr, network.Connected { if _, e = h.Network().DialPeer(ctx, - p.ID); fails(e) { + p.ID); e != nil { continue } diff --git a/pkg/engine/transport/listener.go b/pkg/engine/transport/listener.go index e1f3a744..1fe1edaf 100644 --- a/pkg/engine/transport/listener.go +++ b/pkg/engine/transport/listener.go @@ -4,9 +4,9 @@ import ( "bufio" "context" "fmt" + badger "github.com/indra-labs/go-ds-badger3" "github.com/indra-labs/indra/pkg/engine/protocols" "github.com/indra-labs/indra/pkg/engine/transport/pstoreds" - badger "github.com/ipfs/go-ds-badger" "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host" @@ -33,6 +33,12 @@ type ( // services and relays providing access to them. Listener struct { + // Keys are this node's identity keys, used for identification and + // authentication of peer advertisements. + *crypto.Keys + + *badger.Datastore + // The DHT is used by peer discovery and peer information gossip to // provide information to clients to enable them to set up sessions and // then send traffic through them. @@ -55,10 +61,6 @@ type ( // for a handler to be assigned. newConns chan *Conn - // Keys are this node's identity keys, used for identification and - // authentication of peer advertisements. - *crypto.Keys - // Context here allows listener processes to be signalled to shut down. context.Context @@ -75,9 +77,12 @@ type ( ) // NewListener creates a new Listener with the given parameters. -func NewListener(rendezvous, multiAddr []string, keys *crypto.Keys, store *badger.Datastore, closer func(), ctx context.Context, mtu int, cancel context.CancelFunc) (c *Listener, e error) { +func NewListener(rendezvous, multiAddr []string, keys *crypto.Keys, + store *badger.Datastore, closer func(), ctx context.Context, mtu int, + cancel context.CancelFunc) (c *Listener, e error) { c = &Listener{ + Datastore: store, Keys: keys, MTU: mtu, connections: make(map[string]*Conn), diff --git a/pkg/engine/transport/listener_test.go b/pkg/engine/transport/listener_test.go index dee7bc30..2638557d 100644 --- a/pkg/engine/transport/listener_test.go +++ b/pkg/engine/transport/listener_test.go @@ -2,7 +2,9 @@ package transport import ( "context" + "crypto/rand" "github.com/indra-labs/indra" + "github.com/indra-labs/indra/pkg/crypto/sha256" "os" "testing" "time" @@ -29,7 +31,9 @@ func TestNewListener(t *testing.T) { if err != nil { t.FailNow() } - store, closer := BadgerStore(dataPath) + secret := sha256.New() + rand.Read(secret[:]) + store, closer := BadgerStore(dataPath, secret[:]) if store == nil { t.Fatal("could not open database") } @@ -42,7 +46,9 @@ func TestNewListener(t *testing.T) { if err != nil { t.FailNow() } - store, closer = BadgerStore(dataPath) + secret = sha256.New() + rand.Read(secret[:]) + store, closer = BadgerStore(dataPath, secret[:]) if store == nil { t.Fatal("could not open database") } diff --git a/pkg/engine/transport/pstoreds/ds_test.go b/pkg/engine/transport/pstoreds/ds_test.go index e2044f3f..c900ac7e 100644 --- a/pkg/engine/transport/pstoreds/ds_test.go +++ b/pkg/engine/transport/pstoreds/ds_test.go @@ -10,8 +10,8 @@ import ( pt "github.com/libp2p/go-libp2p/p2p/host/peerstore/test" mockClock "github.com/benbjohnson/clock" + badger "github.com/indra-labs/go-ds-badger3" ds "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger" leveldb "github.com/ipfs/go-ds-leveldb" "github.com/stretchr/testify/require" ) diff --git a/pkg/engine/transport/readme.md b/pkg/engine/transport/readme.md index bf43d933..84e443da 100644 --- a/pkg/engine/transport/readme.md +++ b/pkg/engine/transport/readme.md @@ -2,11 +2,17 @@ This is an implementation of typical network handling features, a listener, which has an `Accept` method that returns a channel that will pick up a new -inbound connection. (todo: is there a proper interface with such a method?) +inbound connection. -(answer to todo: plans afoot to make it a standard net.Listener with the minor -caveat that the `Addr` function only allows one address in return and we support -multiple bound addresses.) +## Warning + +`pstoreds` and `pstoremem` both store the `libp2p.Host`'s private key in +cleartext. Consequently it is necessary to ensure to use `options.Default()` and +use an encryption key with it. The key has to be kept hot for ads, for finding +the LN node being controlled by Indra, and sending/receiving payments. + +todo: need a key change protocol for this identity key that handles session +migration correctly. ## License Notes diff --git a/pkg/p2p/service.go b/pkg/p2p/service.go index 80018502..d3183ee3 100644 --- a/pkg/p2p/service.go +++ b/pkg/p2p/service.go @@ -2,7 +2,9 @@ package p2p import ( "context" + "crypto/rand" "github.com/indra-labs/indra/pkg/cfg" + "github.com/indra-labs/indra/pkg/crypto/sha256" "github.com/indra-labs/indra/pkg/engine/transport" "github.com/indra-labs/indra/pkg/interrupt" "github.com/indra-labs/indra/pkg/p2p/metrics" @@ -61,7 +63,9 @@ func Run() { la = append(la, listenAddresses[i].String()) } var list *transport.Listener - store, closer := transport.BadgerStore(dataPath) + secret := sha256.New() + rand.Read(secret[:]) + store, closer := transport.BadgerStore(dataPath, secret[:]) if store == nil { panic("could not open database") } diff --git a/pkg/storage/service.go b/pkg/storage/service.go index b7bf3cdd..512f10aa 100644 --- a/pkg/storage/service.go +++ b/pkg/storage/service.go @@ -11,7 +11,7 @@ import ( var ( fileName = "indra.db" db *badger.DB - opts badger.Options + opts *badger.Options running sync.Mutex ) @@ -51,8 +51,9 @@ signals: } log.I.Ln("running garbage collection before ready") - db.RunValueLogGC(0.5) - + if e := db.RunValueLogGC(0.5); e != nil { + log.D.Ln(e.Error()) + } log.I.Ln("storage is ready") isReadyChan <- true } diff --git a/pkg/storage/unlock.go b/pkg/storage/unlock.go index f2ba7c5c..30ce581f 100644 --- a/pkg/storage/unlock.go +++ b/pkg/storage/unlock.go @@ -2,17 +2,15 @@ package storage import ( "github.com/dgraph-io/badger/v3" + "github.com/indra-labs/indra/pkg/util/options" "github.com/spf13/viper" ) func attempt_unlock() (isUnlocked bool, err error) { - opts = badger.DefaultOptions(viper.GetString(storeFilePathFlag)) - opts.Logger = nil - opts.IndexCacheSize = 128 << 20 - opts.EncryptionKey = key.Bytes() + opts = options.Default(viper.GetString(storeFilePathFlag), key[:]) - if db, err = badger.Open(opts); err != nil { + if db, err = badger.Open(*opts); err != nil { db = nil diff --git a/pkg/util/options/default.go b/pkg/util/options/default.go new file mode 100644 index 00000000..7d871643 --- /dev/null +++ b/pkg/util/options/default.go @@ -0,0 +1,23 @@ +package options + +import "github.com/dgraph-io/badger/v3" + +// Default returns a pointer to badger.Options to be used to open +// Indra's main data store. +// +// This is separated from the seed's usage of it in order to make test data +// stores without duplicating this common configuration setting. +func Default(filePath string, key []byte) *badger.Options { + + o := badger.DefaultOptions(filePath) + + // If log level is above info maybe we do want this enabled? + o.Logger = nil + + // This works out as 1 << 27, ie 256kb. Should it be 1<<30 1Mb? + o.IndexCacheSize = 128 << 20 + + o.EncryptionKey = key[:] + + return &o +}