mleku/wasmd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Monitor WasmVM cache metrics (#503)

Browse Source 
* Start collect WasmVM cache metrics

* Review comments

* Integrate wasmvm metrics

* Bring all together
This commit is contained in:
Alexander Peters
2021-04-28 15:08:54 +02:00
committed by GitHub
parent 846611bcd2
commit 70dedadb9c
8 changed files with 121 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/wasmd/root.go
View File

@@ -2,6 +2,8 @@ package main
import (
"errors"
wasmkeeper "github.com/CosmWasm/wasmd/x/wasm/keeper"
"github.com/prometheus/client_golang/prometheus"
"io"
"os"
"path/filepath"
@@ -199,13 +201,17 @@ func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, appOpts serverty
if err != nil {
panic(err)
}
var emptyWasmOpts []wasm.Option
var wasmOpts []wasm.Option
if cast.ToBool(appOpts.Get("telemetry.enabled")) {
wasmOpts = append(wasmOpts, wasmkeeper.WithVMCacheMetrics(prometheus.DefaultRegisterer))
}
return app.NewWasmApp(logger, db, traceStore, true, skipUpgradeHeights,
cast.ToString(appOpts.Get(flags.FlagHome)),
cast.ToUint(appOpts.Get(server.FlagInvCheckPeriod)),
app.GetEnabledProposals(),
appOpts,
emptyWasmOpts,
wasmOpts,
baseapp.SetPruning(pruningOpts),
baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))),
baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))),

8
contrib/prometheus/README.md
View File

@@ -18,7 +18,6 @@ prometheus-retention-time = 15
`retention-time` must be >0 (see prometheus scrape config)
* Edit `$HOME/config/config.toml`
```toml
[instrumentation]
@@ -28,6 +27,13 @@ prometheus-retention-time = 15
# Check out the documentation for the list of available metrics.
prometheus = true
```
Test manually at:
`http://localhost:1317/metrics?format=prometheus`
Note the `format` parameter in the request for the endpoint:
# Local testing
## Run Prometheus
```sh

2
go.mod
View File

@@ -13,6 +13,8 @@ require (
github.com/gorilla/mux v1.8.0
github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.10.0
github.com/prometheus/common v0.21.0 // indirect
github.com/rakyll/statik v0.1.7
github.com/regen-network/cosmos-proto v0.3.1
github.com/rs/zerolog v1.21.0

16
go.sum
View File

@@ -220,6 +220,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@@ -445,8 +446,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw=
github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM=
github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg=
github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -462,16 +464,19 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM=
github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/common v0.21.0 h1:SMvI2JVldvfUvRVlP64jkIJEC6WiGHJcN2e5tB+ztF8=
github.com/prometheus/common v0.21.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
@@ -693,6 +698,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -733,8 +739,10 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

72
x/wasm/keeper/metrics.go Normal file
View File

@@ -0,0 +1,72 @@
package keeper
import (
wasmvmtypes "github.com/CosmWasm/wasmvm/types"
"github.com/prometheus/client_golang/prometheus"
)
const (
labelPinned = "pinned"
labelMemory = "memory"
labelFs = "fs"
)
// metricSource source of wasmvm metrics
type metricSource interface {
GetMetrics() (*wasmvmtypes.Metrics, error)
}
var _ prometheus.Collector = (*WasmVMMetricsCollector)(nil)
// WasmVMMetricsCollector custom metrics collector to be used with Prometheus
type WasmVMMetricsCollector struct {
source metricSource
CacheHitsDescr *prometheus.Desc
CacheMissesDescr *prometheus.Desc
CacheElementsDescr *prometheus.Desc
CacheSizeDescr *prometheus.Desc
}
//NewWasmVMMetricsCollector constructor
func NewWasmVMMetricsCollector(s metricSource) *WasmVMMetricsCollector {
return &WasmVMMetricsCollector{
source: s,
CacheHitsDescr: prometheus.NewDesc("wasmvm_cache_hits_total", "Total number of cache hits", []string{"type"}, nil),
CacheMissesDescr: prometheus.NewDesc("wasmvm_cache_misses_total", "Total number of cache misses", nil, nil),
CacheElementsDescr: prometheus.NewDesc("wasmvm_cache_elements_total", "Total number of elements in the cache", []string{"type"}, nil),
CacheSizeDescr: prometheus.NewDesc("wasmvm_cache_size_bytes", "Total number of elements in the cache", []string{"type"}, nil),
}
}
// Register registers all metrics
func (p *WasmVMMetricsCollector) Register(r prometheus.Registerer) {
r.MustRegister(p)
}
// Describe sends the super-set of all possible descriptors of metrics
func (p *WasmVMMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
descs <- p.CacheHitsDescr
descs <- p.CacheMissesDescr
descs <- p.CacheElementsDescr
descs <- p.CacheSizeDescr
}
// Collect is called by the Prometheus registry when collecting metrics.
func (p *WasmVMMetricsCollector) Collect(c chan<- prometheus.Metric) {
m, err := p.source.GetMetrics()
if err != nil {
return
}
c <- prometheus.MustNewConstMetric(p.CacheHitsDescr, prometheus.CounterValue, float64(m.HitsPinnedMemoryCache), labelPinned)
c <- prometheus.MustNewConstMetric(p.CacheHitsDescr, prometheus.CounterValue, float64(m.HitsMemoryCache), labelMemory)
c <- prometheus.MustNewConstMetric(p.CacheHitsDescr, prometheus.CounterValue, float64(m.HitsFsCache), labelFs)
c <- prometheus.MustNewConstMetric(p.CacheMissesDescr, prometheus.CounterValue, float64(m.Misses))
c <- prometheus.MustNewConstMetric(p.CacheElementsDescr, prometheus.GaugeValue, float64(m.ElementsPinnedMemoryCache), labelPinned)
c <- prometheus.MustNewConstMetric(p.CacheElementsDescr, prometheus.GaugeValue, float64(m.ElementsMemoryCache), labelMemory)
c <- prometheus.MustNewConstMetric(p.CacheSizeDescr, prometheus.GaugeValue, float64(m.SizeMemoryCache), labelMemory)
c <- prometheus.MustNewConstMetric(p.CacheSizeDescr, prometheus.GaugeValue, float64(m.SizePinnedMemoryCache), labelPinned)
// Node about fs metrics:
// The number of elements and the size of elements in the file system cache cannot easily be obtained.
// We had to either scan the whole directory of potentially thousands of files or track the values when files are added or removed.
// Such a tracking would need to be on disk such that the values are not cleared when the node is restarted.
}

7
x/wasm/keeper/options.go
View File

@@ -3,6 +3,7 @@ package keeper
import (
"fmt"
"github.com/CosmWasm/wasmd/x/wasm/types"
"github.com/prometheus/client_golang/prometheus"
)
type optsFn func(*Keeper)
@@ -74,3 +75,9 @@ func WithCoinTransferrer(x CoinTransferrer) Option {
k.bank = x
})
}
func WithVMCacheMetrics(r prometheus.Registerer) Option {
return optsFn(func(k *Keeper) {
NewWasmVMMetricsCollector(k.wasmVM).Register(r)
})
}

10
x/wasm/keeper/wasmtesting/mock_engine.go
View File

@@ -33,6 +33,7 @@ type MockWasmer struct {
IBCPacketTimeoutFn func(codeID wasmvm.Checksum, env wasmvmtypes.Env, packet wasmvmtypes.IBCPacket, store wasmvm.KVStore, goapi wasmvm.GoAPI, querier wasmvm.Querier, gasMeter wasmvm.GasMeter, gasLimit uint64) (*wasmvmtypes.IBCBasicResponse, uint64, error)
PinFn func(checksum wasmvm.Checksum) error
UnpinFn func(checksum wasmvm.Checksum) error
GetMetricsFn func() (*wasmvmtypes.Metrics, error)
}
func (m *MockWasmer) IBCChannelOpen(codeID wasmvm.Checksum, env wasmvmtypes.Env, channel wasmvmtypes.IBCChannel, store wasmvm.KVStore, goapi wasmvm.GoAPI, querier wasmvm.Querier, gasMeter wasmvm.GasMeter, gasLimit uint64) (uint64, error) {
@@ -162,9 +163,16 @@ func (m *MockWasmer) Unpin(checksum wasmvm.Checksum) error {
return m.UnpinFn(checksum)
}
func (m *MockWasmer) GetMetrics() (*wasmvmtypes.Metrics, error) {
if m.GetMetricsFn == nil {
panic("not expected to be called")
}
return m.GetMetricsFn()
}
var AlwaysPanicMockWasmer = &MockWasmer{}
// selfCallingInstMockWasmer prepares a Wasmer mock that calls itself on instantiation.
// SelfCallingInstMockWasmer prepares a Wasmer mock that calls itself on instantiation.
func SelfCallingInstMockWasmer(executeCalled *bool) *MockWasmer {
return &MockWasmer{

5
x/wasm/types/wasmer_engine.go
View File

@@ -18,7 +18,7 @@ type WasmerEngine interface {
// be instantiated with custom inputs in the future.
Create(code wasmvm.WasmCode) (wasmvm.Checksum, error)
// This will statically analyze the code.
// AnalyzeCode will statically analyze the code.
// Currently just reports if it exposes all IBC entry points.
AnalyzeCode(checksum wasmvm.Checksum) (*wasmvmtypes.AnalysisReport, error)
@@ -220,4 +220,7 @@ type WasmerEngine interface {
// the implementor's choice.
// Unpin is idempotent.
Unpin(checksum wasmvm.Checksum) error
// GetMetrics some internal metrics for monitoring purposes.
GetMetrics() (*wasmvmtypes.Metrics, error)
}