diff --git a/cmd/wasmd/root.go b/cmd/wasmd/root.go index f8bca81e..c2f5700a 100644 --- a/cmd/wasmd/root.go +++ b/cmd/wasmd/root.go @@ -2,6 +2,8 @@ package main import ( "errors" + wasmkeeper "github.com/CosmWasm/wasmd/x/wasm/keeper" + "github.com/prometheus/client_golang/prometheus" "io" "os" "path/filepath" @@ -199,13 +201,17 @@ func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, appOpts serverty if err != nil { panic(err) } - var emptyWasmOpts []wasm.Option + var wasmOpts []wasm.Option + if cast.ToBool(appOpts.Get("telemetry.enabled")) { + wasmOpts = append(wasmOpts, wasmkeeper.WithVMCacheMetrics(prometheus.DefaultRegisterer)) + } + return app.NewWasmApp(logger, db, traceStore, true, skipUpgradeHeights, cast.ToString(appOpts.Get(flags.FlagHome)), cast.ToUint(appOpts.Get(server.FlagInvCheckPeriod)), app.GetEnabledProposals(), appOpts, - emptyWasmOpts, + wasmOpts, baseapp.SetPruning(pruningOpts), baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))), baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))), diff --git a/contrib/prometheus/README.md b/contrib/prometheus/README.md index 7c8d36ea..398973c6 100644 --- a/contrib/prometheus/README.md +++ b/contrib/prometheus/README.md @@ -18,7 +18,6 @@ prometheus-retention-time = 15 `retention-time` must be >0 (see prometheus scrape config) - * Edit `$HOME/config/config.toml` ```toml [instrumentation] @@ -28,6 +27,13 @@ prometheus-retention-time = 15 # Check out the documentation for the list of available metrics. prometheus = true ``` + +Test manually at: +`http://localhost:1317/metrics?format=prometheus` + +Note the `format` parameter in the request for the endpoint: + + # Local testing ## Run Prometheus ```sh diff --git a/go.mod b/go.mod index 30182d12..d3d221cd 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,8 @@ require ( github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.10.0 + github.com/prometheus/common v0.21.0 // indirect github.com/rakyll/statik v0.1.7 github.com/regen-network/cosmos-proto v0.3.1 github.com/rs/zerolog v1.21.0 diff --git a/go.sum b/go.sum index 17501578..cc213ff3 100644 --- a/go.sum +++ b/go.sum @@ -220,6 +220,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -445,8 +446,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw= github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= +github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= +github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -462,16 +464,19 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.21.0 h1:SMvI2JVldvfUvRVlP64jkIJEC6WiGHJcN2e5tB+ztF8= +github.com/prometheus/common v0.21.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= @@ -693,6 +698,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -733,8 +739,10 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/x/wasm/keeper/metrics.go b/x/wasm/keeper/metrics.go new file mode 100644 index 00000000..d271335e --- /dev/null +++ b/x/wasm/keeper/metrics.go @@ -0,0 +1,72 @@ +package keeper + +import ( + wasmvmtypes "github.com/CosmWasm/wasmvm/types" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + labelPinned = "pinned" + labelMemory = "memory" + labelFs = "fs" +) + +// metricSource source of wasmvm metrics +type metricSource interface { + GetMetrics() (*wasmvmtypes.Metrics, error) +} + +var _ prometheus.Collector = (*WasmVMMetricsCollector)(nil) + +// WasmVMMetricsCollector custom metrics collector to be used with Prometheus +type WasmVMMetricsCollector struct { + source metricSource + CacheHitsDescr *prometheus.Desc + CacheMissesDescr *prometheus.Desc + CacheElementsDescr *prometheus.Desc + CacheSizeDescr *prometheus.Desc +} + +//NewWasmVMMetricsCollector constructor +func NewWasmVMMetricsCollector(s metricSource) *WasmVMMetricsCollector { + return &WasmVMMetricsCollector{ + source: s, + CacheHitsDescr: prometheus.NewDesc("wasmvm_cache_hits_total", "Total number of cache hits", []string{"type"}, nil), + CacheMissesDescr: prometheus.NewDesc("wasmvm_cache_misses_total", "Total number of cache misses", nil, nil), + CacheElementsDescr: prometheus.NewDesc("wasmvm_cache_elements_total", "Total number of elements in the cache", []string{"type"}, nil), + CacheSizeDescr: prometheus.NewDesc("wasmvm_cache_size_bytes", "Total number of elements in the cache", []string{"type"}, nil), + } +} + +// Register registers all metrics +func (p *WasmVMMetricsCollector) Register(r prometheus.Registerer) { + r.MustRegister(p) +} + +// Describe sends the super-set of all possible descriptors of metrics +func (p *WasmVMMetricsCollector) Describe(descs chan<- *prometheus.Desc) { + descs <- p.CacheHitsDescr + descs <- p.CacheMissesDescr + descs <- p.CacheElementsDescr + descs <- p.CacheSizeDescr +} + +// Collect is called by the Prometheus registry when collecting metrics. +func (p *WasmVMMetricsCollector) Collect(c chan<- prometheus.Metric) { + m, err := p.source.GetMetrics() + if err != nil { + return + } + c <- prometheus.MustNewConstMetric(p.CacheHitsDescr, prometheus.CounterValue, float64(m.HitsPinnedMemoryCache), labelPinned) + c <- prometheus.MustNewConstMetric(p.CacheHitsDescr, prometheus.CounterValue, float64(m.HitsMemoryCache), labelMemory) + c <- prometheus.MustNewConstMetric(p.CacheHitsDescr, prometheus.CounterValue, float64(m.HitsFsCache), labelFs) + c <- prometheus.MustNewConstMetric(p.CacheMissesDescr, prometheus.CounterValue, float64(m.Misses)) + c <- prometheus.MustNewConstMetric(p.CacheElementsDescr, prometheus.GaugeValue, float64(m.ElementsPinnedMemoryCache), labelPinned) + c <- prometheus.MustNewConstMetric(p.CacheElementsDescr, prometheus.GaugeValue, float64(m.ElementsMemoryCache), labelMemory) + c <- prometheus.MustNewConstMetric(p.CacheSizeDescr, prometheus.GaugeValue, float64(m.SizeMemoryCache), labelMemory) + c <- prometheus.MustNewConstMetric(p.CacheSizeDescr, prometheus.GaugeValue, float64(m.SizePinnedMemoryCache), labelPinned) + // Node about fs metrics: + // The number of elements and the size of elements in the file system cache cannot easily be obtained. + // We had to either scan the whole directory of potentially thousands of files or track the values when files are added or removed. + // Such a tracking would need to be on disk such that the values are not cleared when the node is restarted. +} diff --git a/x/wasm/keeper/options.go b/x/wasm/keeper/options.go index 8d9872d8..85aa835f 100644 --- a/x/wasm/keeper/options.go +++ b/x/wasm/keeper/options.go @@ -3,6 +3,7 @@ package keeper import ( "fmt" "github.com/CosmWasm/wasmd/x/wasm/types" + "github.com/prometheus/client_golang/prometheus" ) type optsFn func(*Keeper) @@ -74,3 +75,9 @@ func WithCoinTransferrer(x CoinTransferrer) Option { k.bank = x }) } + +func WithVMCacheMetrics(r prometheus.Registerer) Option { + return optsFn(func(k *Keeper) { + NewWasmVMMetricsCollector(k.wasmVM).Register(r) + }) +} diff --git a/x/wasm/keeper/wasmtesting/mock_engine.go b/x/wasm/keeper/wasmtesting/mock_engine.go index 0e8fbb0c..dc8a4c8c 100644 --- a/x/wasm/keeper/wasmtesting/mock_engine.go +++ b/x/wasm/keeper/wasmtesting/mock_engine.go @@ -33,6 +33,7 @@ type MockWasmer struct { IBCPacketTimeoutFn func(codeID wasmvm.Checksum, env wasmvmtypes.Env, packet wasmvmtypes.IBCPacket, store wasmvm.KVStore, goapi wasmvm.GoAPI, querier wasmvm.Querier, gasMeter wasmvm.GasMeter, gasLimit uint64) (*wasmvmtypes.IBCBasicResponse, uint64, error) PinFn func(checksum wasmvm.Checksum) error UnpinFn func(checksum wasmvm.Checksum) error + GetMetricsFn func() (*wasmvmtypes.Metrics, error) } func (m *MockWasmer) IBCChannelOpen(codeID wasmvm.Checksum, env wasmvmtypes.Env, channel wasmvmtypes.IBCChannel, store wasmvm.KVStore, goapi wasmvm.GoAPI, querier wasmvm.Querier, gasMeter wasmvm.GasMeter, gasLimit uint64) (uint64, error) { @@ -162,9 +163,16 @@ func (m *MockWasmer) Unpin(checksum wasmvm.Checksum) error { return m.UnpinFn(checksum) } +func (m *MockWasmer) GetMetrics() (*wasmvmtypes.Metrics, error) { + if m.GetMetricsFn == nil { + panic("not expected to be called") + } + return m.GetMetricsFn() +} + var AlwaysPanicMockWasmer = &MockWasmer{} -// selfCallingInstMockWasmer prepares a Wasmer mock that calls itself on instantiation. +// SelfCallingInstMockWasmer prepares a Wasmer mock that calls itself on instantiation. func SelfCallingInstMockWasmer(executeCalled *bool) *MockWasmer { return &MockWasmer{ diff --git a/x/wasm/types/wasmer_engine.go b/x/wasm/types/wasmer_engine.go index 501d6871..7a7d08f7 100644 --- a/x/wasm/types/wasmer_engine.go +++ b/x/wasm/types/wasmer_engine.go @@ -18,7 +18,7 @@ type WasmerEngine interface { // be instantiated with custom inputs in the future. Create(code wasmvm.WasmCode) (wasmvm.Checksum, error) - // This will statically analyze the code. + // AnalyzeCode will statically analyze the code. // Currently just reports if it exposes all IBC entry points. AnalyzeCode(checksum wasmvm.Checksum) (*wasmvmtypes.AnalysisReport, error) @@ -220,4 +220,7 @@ type WasmerEngine interface { // the implementor's choice. // Unpin is idempotent. Unpin(checksum wasmvm.Checksum) error + + // GetMetrics some internal metrics for monitoring purposes. + GetMetrics() (*wasmvmtypes.Metrics, error) }