Start system tests (#1410)

* Start system tests

* Go mod tidy

* Add system tests (#1411)

* Add tests

* Add test-system to CI

* Fix path

* Remove store artifact steps

* Add small fixes

* Add stake/unstake tests

* Add unsafe-reset-all extention + system test

* Replace ustake with stake

* Add more tests + fix bug in cli

* Fix comments and add multi contract system test

* Updates and fixes to system tests (#1449)

* Updates

* Minor cleanup

* Make tests pass

---------

Co-authored-by: Alexander Peters <alpe@users.noreply.github.com>

* Fix Makefile to return exit code for system tests (#1450)

* Abort on error results

---------

Co-authored-by: pinosu <95283998+pinosu@users.noreply.github.com>

.circleci/config.yml

@@ -96,6 +96,20 @@ jobs:
       - store_artifacts:
           path: /tmp/logs
   
+  test-system:
+    executor: golang
+    parallelism: 1
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - restore_cache:
+          keys:
+            - go-mod-v1-{{ checksum "go.sum" }}
+      - run:
+          name: Build and run system tests
+          command: make test-system
+
   benchmark:
     executor: golang
     parallelism: 1
@@ -234,6 +248,9 @@ workflows:
       - upload-coverage:
           requires:
             - test-cover
+      - test-system:
+          requires:
+            - test-cover
       - benchmark:
           requires:
             - test-cover

Makefile

@@ -124,9 +124,8 @@ distclean: clean
 ########################################
 ### Testing
 
-
 test: test-unit
-test-all: check test-race test-cover
+test-all: check test-race test-cover test-system
 
 test-unit:
 	@VERSION=$(VERSION) go test -mod=readonly -tags='ledger test_ledger_mock' ./...
@@ -152,6 +151,9 @@ test-sim-deterministic: runsim
 	@echo "Running short multi-seed application simulation. This may take awhile!"
 	@$(BINDIR)/runsim -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 1 1 TestAppStateDeterminism
 
+test-system: install
+	@VERSION=$(VERSION) cd tests/system; go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=45s --verbose; EXIT_CODE=$$?; cd -; exit $$EXIT_CODE
+
 ###############################################################################
 ###                                Linting                                  ###
 ###############################################################################
@@ -201,3 +203,4 @@ proto-check-breaking:
 	go-mod-cache draw-deps clean build format \
 	test test-all test-build test-cover test-unit test-race \
 	test-sim-import-export build-windows-client \
+	test-system

cmd/wasmd/root.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"os"
 
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
+
 	rosettaCmd "cosmossdk.io/tools/rosetta/cmd"
 	dbm "github.com/cometbft/cometbft-db"
 	tmcfg "github.com/cometbft/cometbft/config"
@@ -33,6 +35,7 @@ import (
 	"github.com/CosmWasm/wasmd/app"
 	"github.com/CosmWasm/wasmd/app/params"
 	"github.com/CosmWasm/wasmd/x/wasm"
+	wasmcli "github.com/CosmWasm/wasmd/x/wasm/client/cli"
 	wasmkeeper "github.com/CosmWasm/wasmd/x/wasm/keeper"
 	wasmtypes "github.com/CosmWasm/wasmd/x/wasm/types"
 )
@@ -148,13 +151,14 @@ func initAppConfig() (string, interface{}) {
 func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
 	rootCmd.AddCommand(
 		genutilcli.InitCmd(app.ModuleBasics, app.DefaultNodeHome),
-		// testnetCmd(app.ModuleBasics, banktypes.GenesisBalancesIterator{}),
+		NewTestnetCmd(app.ModuleBasics, banktypes.GenesisBalancesIterator{}),
 		debug.Cmd(),
 		config.Cmd(),
 		pruning.PruningCmd(newApp),
 	)
 
 	server.AddCommands(rootCmd, app.DefaultNodeHome, newApp, appExport, addModuleInitFlags)
+	wasmcli.ExtendUnsafeResetAllCmd(rootCmd)
 
 	// add keybase, auxiliary RPC, query, and tx child commands
 	rootCmd.AddCommand(

cmd/wasmd/testnet.go

@@ -0,0 +1,580 @@
+package main
+
+// DONTCOVER
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/cosmos/cosmos-sdk/version"
+
+	"github.com/CosmWasm/wasmd/app"
+
+	tmconfig "github.com/cometbft/cometbft/config"
+	tmrand "github.com/cometbft/cometbft/libs/rand"
+	"github.com/cometbft/cometbft/types"
+	tmtime "github.com/cometbft/cometbft/types/time"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"cosmossdk.io/math"
+
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/cosmos/cosmos-sdk/client/flags"
+	"github.com/cosmos/cosmos-sdk/client/tx"
+	"github.com/cosmos/cosmos-sdk/crypto/hd"
+	"github.com/cosmos/cosmos-sdk/crypto/keyring"
+	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
+	"github.com/cosmos/cosmos-sdk/server"
+	srvconfig "github.com/cosmos/cosmos-sdk/server/config"
+	"github.com/cosmos/cosmos-sdk/testutil"
+	"github.com/cosmos/cosmos-sdk/testutil/network"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/types/module"
+	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
+	"github.com/cosmos/cosmos-sdk/x/genutil"
+	genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
+	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+)
+
+var (
+	flagNodeDirPrefix     = "node-dir-prefix"
+	flagNumValidators     = "v"
+	flagOutputDir         = "output-dir"
+	flagNodeDaemonHome    = "node-daemon-home"
+	flagStartingIPAddress = "starting-ip-address"
+	flagEnableLogging     = "enable-logging"
+	flagGRPCAddress       = "grpc.address"
+	flagRPCAddress        = "rpc.address"
+	flagAPIAddress        = "api.address"
+	flagPrintMnemonic     = "print-mnemonic"
+	// custom flags
+	flagCommitTimeout = "commit-timeout"
+	flagSingleHost    = "single-host"
+)
+
+type initArgs struct {
+	algo              string
+	chainID           string
+	keyringBackend    string
+	minGasPrices      string
+	nodeDaemonHome    string
+	nodeDirPrefix     string
+	numValidators     int
+	outputDir         string
+	startingIPAddress string
+	singleMachine     bool
+}
+
+type startArgs struct {
+	algo          string
+	apiAddress    string
+	chainID       string
+	enableLogging bool
+	grpcAddress   string
+	minGasPrices  string
+	numValidators int
+	outputDir     string
+	printMnemonic bool
+	rpcAddress    string
+}
+
+func addTestnetFlagsToCmd(cmd *cobra.Command) {
+	cmd.Flags().Int(flagNumValidators, 4, "Number of validators to initialize the testnet with")
+	cmd.Flags().StringP(flagOutputDir, "o", "./.testnets", "Directory to store initialization data for the testnet")
+	cmd.Flags().String(flags.FlagChainID, "", "genesis file chain-id, if left blank will be randomly created")
+	cmd.Flags().String(server.FlagMinGasPrices, fmt.Sprintf("0.000006%s", sdk.DefaultBondDenom), "Minimum gas prices to accept for transactions; All fees in a tx must meet this minimum (e.g. 0.01photino,0.001stake)")
+	cmd.Flags().String(flags.FlagKeyType, string(hd.Secp256k1Type), "Key signing algorithm to generate keys for")
+
+	// support old flags name for backwards compatibility
+	cmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName {
+		if name == "algo" {
+			name = flags.FlagKeyType
+		}
+
+		return pflag.NormalizedName(name)
+	})
+}
+
+// NewTestnetCmd creates a root testnet command with subcommands to run an in-process testnet or initialize
+// validator configuration files for running a multi-validator testnet in a separate process
+func NewTestnetCmd(mbm module.BasicManager, genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
+	testnetCmd := &cobra.Command{
+		Use:                        "testnet",
+		Short:                      "subcommands for starting or configuring local testnets",
+		DisableFlagParsing:         true,
+		SuggestionsMinimumDistance: 2,
+		RunE:                       client.ValidateCmd,
+	}
+
+	testnetCmd.AddCommand(testnetStartCmd())
+	testnetCmd.AddCommand(testnetInitFilesCmd(mbm, genBalIterator))
+
+	return testnetCmd
+}
+
+// testnetInitFilesCmd returns a cmd to initialize all files for tendermint testnet and application
+func testnetInitFilesCmd(mbm module.BasicManager, genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "init-files",
+		Short: "Initialize config directories & files for a multi-validator testnet running locally via separate processes (e.g. Docker Compose or similar)",
+		Long: fmt.Sprintf(`init-files will setup "v" number of directories and populate each with
+necessary files (private validator, genesis, config, etc.) for running "v" validator nodes.
+
+Booting up a network with these validator folders is intended to be used with Docker Compose,
+or a similar setup where each node has a manually configurable IP address.
+
+Note, strict routability for addresses is turned off in the config file.
+
+Example:
+	%s testnet init-files --v 4 --output-dir ./.testnets --starting-ip-address 192.168.10.2
+	`, version.AppName),
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			clientCtx, err := client.GetClientQueryContext(cmd)
+			if err != nil {
+				return err
+			}
+
+			serverCtx := server.GetServerContextFromCmd(cmd)
+			config := serverCtx.Config
+
+			args := initArgs{}
+			args.outputDir, _ = cmd.Flags().GetString(flagOutputDir)
+			args.keyringBackend, _ = cmd.Flags().GetString(flags.FlagKeyringBackend)
+			args.chainID, _ = cmd.Flags().GetString(flags.FlagChainID)
+			args.minGasPrices, _ = cmd.Flags().GetString(server.FlagMinGasPrices)
+			args.nodeDirPrefix, _ = cmd.Flags().GetString(flagNodeDirPrefix)
+			args.nodeDaemonHome, _ = cmd.Flags().GetString(flagNodeDaemonHome)
+			args.startingIPAddress, _ = cmd.Flags().GetString(flagStartingIPAddress)
+			args.numValidators, _ = cmd.Flags().GetInt(flagNumValidators)
+			args.algo, _ = cmd.Flags().GetString(flags.FlagKeyType)
+
+			args.singleMachine, _ = cmd.Flags().GetBool(flagSingleHost)
+			config.Consensus.TimeoutCommit, err = cmd.Flags().GetDuration(flagCommitTimeout)
+			if err != nil {
+				return err
+			}
+
+			return initTestnetFiles(clientCtx, cmd, config, mbm, genBalIterator, args)
+		},
+	}
+
+	addTestnetFlagsToCmd(cmd)
+	cmd.Flags().String(flagNodeDirPrefix, "node", "Prefix the directory name for each node with (node results in node0, node1, ...)")
+	cmd.Flags().String(flagNodeDaemonHome, "wasmd", "Home directory of the node's daemon configuration")
+	cmd.Flags().String(flagStartingIPAddress, "192.168.0.1", "Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:46656, ID1@192.168.0.2:46656, ...)")
+	cmd.Flags().String(flags.FlagKeyringBackend, flags.DefaultKeyringBackend, "Select keyring's backend (os|file|test)")
+	cmd.Flags().Duration(flagCommitTimeout, 5*time.Second, "Time to wait after a block commit before starting on the new height")
+	cmd.Flags().Bool(flagSingleHost, false, "Cluster runs on a single host machine with different ports")
+
+	return cmd
+}
+
+// testnetStartCmd returns a cmd to start multi validator in-process testnet
+func testnetStartCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "start",
+		Short: "Launch an in-process multi-validator testnet",
+		Long: fmt.Sprintf(`testnet will launch an in-process multi-validator testnet,
+and generate "v" directories, populated with necessary validator configuration files
+(private validator, genesis, config, etc.).
+
+Example:
+	%s testnet --v 4 --output-dir ./.testnets
+	`, version.AppName),
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			args := startArgs{}
+			args.outputDir, _ = cmd.Flags().GetString(flagOutputDir)
+			args.chainID, _ = cmd.Flags().GetString(flags.FlagChainID)
+			args.minGasPrices, _ = cmd.Flags().GetString(server.FlagMinGasPrices)
+			args.numValidators, _ = cmd.Flags().GetInt(flagNumValidators)
+			args.algo, _ = cmd.Flags().GetString(flags.FlagKeyType)
+			args.enableLogging, _ = cmd.Flags().GetBool(flagEnableLogging)
+			args.rpcAddress, _ = cmd.Flags().GetString(flagRPCAddress)
+			args.apiAddress, _ = cmd.Flags().GetString(flagAPIAddress)
+			args.grpcAddress, _ = cmd.Flags().GetString(flagGRPCAddress)
+			args.printMnemonic, _ = cmd.Flags().GetBool(flagPrintMnemonic)
+
+			return startTestnet(cmd, args)
+		},
+	}
+
+	addTestnetFlagsToCmd(cmd)
+	cmd.Flags().Bool(flagEnableLogging, false, "Enable INFO logging of tendermint validator nodes")
+	cmd.Flags().String(flagRPCAddress, "tcp://0.0.0.0:26657", "the RPC address to listen on")
+	cmd.Flags().String(flagAPIAddress, "tcp://0.0.0.0:1317", "the address to listen on for REST API")
+	cmd.Flags().String(flagGRPCAddress, "0.0.0.0:9090", "the gRPC server address to listen on")
+	cmd.Flags().Bool(flagPrintMnemonic, true, "print mnemonic of first validator to stdout for manual testing")
+	return cmd
+}
+
+const nodeDirPerm = 0o755
+
+// initTestnetFiles initializes testnet files for a testnet to be run in a separate process
+func initTestnetFiles(
+	clientCtx client.Context,
+	cmd *cobra.Command,
+	nodeConfig *tmconfig.Config,
+	mbm module.BasicManager,
+	genBalIterator banktypes.GenesisBalancesIterator,
+	args initArgs,
+) error {
+	if args.chainID == "" {
+		args.chainID = "chain-" + tmrand.Str(6)
+	}
+	nodeIDs := make([]string, args.numValidators)
+	valPubKeys := make([]cryptotypes.PubKey, args.numValidators)
+
+	appConfig := srvconfig.DefaultConfig()
+	appConfig.MinGasPrices = args.minGasPrices
+	appConfig.API.Enable = true
+	appConfig.Telemetry.Enabled = true
+	appConfig.Telemetry.PrometheusRetentionTime = 60
+	appConfig.Telemetry.EnableHostnameLabel = false
+	appConfig.Telemetry.GlobalLabels = [][]string{{"chain_id", args.chainID}}
+
+	var (
+		genAccounts []authtypes.GenesisAccount
+		genBalances []banktypes.Balance
+		genFiles    []string
+	)
+	const (
+		rpcPort     = 26657
+		apiPort     = 1317
+		grpcPort    = 9090
+		grpcWebPort = 8090
+	)
+	p2pPortStart := 26656
+
+	inBuf := bufio.NewReader(cmd.InOrStdin())
+	// generate private keys, node IDs, and initial transactions
+	for i := 0; i < args.numValidators; i++ {
+		var portOffset int
+		if args.singleMachine {
+			portOffset = i
+			p2pPortStart = 16656 // use different start point to not conflict with rpc port
+			nodeConfig.P2P.AddrBookStrict = false
+			nodeConfig.P2P.PexReactor = false
+			nodeConfig.P2P.AllowDuplicateIP = true
+		}
+
+		nodeDirName := fmt.Sprintf("%s%d", args.nodeDirPrefix, i)
+		nodeDir := filepath.Join(args.outputDir, nodeDirName, args.nodeDaemonHome)
+		gentxsDir := filepath.Join(args.outputDir, "gentxs")
+
+		nodeConfig.SetRoot(nodeDir)
+		nodeConfig.Moniker = nodeDirName
+		nodeConfig.RPC.ListenAddress = "tcp://0.0.0.0:26657"
+
+		appConfig.API.Address = fmt.Sprintf("tcp://0.0.0.0:%d", apiPort+portOffset)
+		appConfig.GRPC.Address = fmt.Sprintf("0.0.0.0:%d", grpcPort+portOffset)
+		appConfig.GRPCWeb.Address = fmt.Sprintf("0.0.0.0:%d", grpcWebPort+portOffset)
+
+		if err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm); err != nil {
+			_ = os.RemoveAll(args.outputDir)
+			return err
+		}
+
+		ip, err := getIP(i, args.startingIPAddress)
+		if err != nil {
+			_ = os.RemoveAll(args.outputDir)
+			return err
+		}
+
+		nodeIDs[i], valPubKeys[i], err = genutil.InitializeNodeValidatorFiles(nodeConfig)
+		if err != nil {
+			_ = os.RemoveAll(args.outputDir)
+			return err
+		}
+
+		memo := fmt.Sprintf("%s@%s:%d", nodeIDs[i], ip, p2pPortStart+portOffset)
+		genFiles = append(genFiles, nodeConfig.GenesisFile())
+
+		kb, err := keyring.New(sdk.KeyringServiceName(), args.keyringBackend, nodeDir, inBuf, clientCtx.Codec)
+		if err != nil {
+			return err
+		}
+
+		keyringAlgos, _ := kb.SupportedAlgorithms()
+		algo, err := keyring.NewSigningAlgoFromString(args.algo, keyringAlgos)
+		if err != nil {
+			return err
+		}
+
+		addr, secret, err := testutil.GenerateSaveCoinKey(kb, nodeDirName, "", true, algo)
+		if err != nil {
+			_ = os.RemoveAll(args.outputDir)
+			return err
+		}
+
+		info := map[string]string{"secret": secret}
+
+		cliPrint, err := json.Marshal(info)
+		if err != nil {
+			return err
+		}
+
+		// save private key seed words
+		if err := writeFile(fmt.Sprintf("%v.json", "key_seed"), nodeDir, cliPrint); err != nil {
+			return err
+		}
+
+		accTokens := sdk.TokensFromConsensusPower(1000, sdk.DefaultPowerReduction)
+		accStakingTokens := sdk.TokensFromConsensusPower(500, sdk.DefaultPowerReduction)
+		coins := sdk.Coins{
+			sdk.NewCoin("testtoken", accTokens),
+			sdk.NewCoin(sdk.DefaultBondDenom, accStakingTokens),
+		}
+
+		genBalances = append(genBalances, banktypes.Balance{Address: addr.String(), Coins: coins.Sort()})
+		genAccounts = append(genAccounts, authtypes.NewBaseAccount(addr, nil, 0, 0))
+
+		valTokens := sdk.TokensFromConsensusPower(100, sdk.DefaultPowerReduction)
+		createValMsg, err := stakingtypes.NewMsgCreateValidator(
+			sdk.ValAddress(addr),
+			valPubKeys[i],
+			sdk.NewCoin(sdk.DefaultBondDenom, valTokens),
+			stakingtypes.NewDescription(nodeDirName, "", "", "", ""),
+			stakingtypes.NewCommissionRates(math.LegacyOneDec(), math.LegacyOneDec(), math.LegacyOneDec()),
+			math.OneInt(),
+		)
+		if err != nil {
+			return err
+		}
+
+		txBuilder := clientCtx.TxConfig.NewTxBuilder()
+		if err := txBuilder.SetMsgs(createValMsg); err != nil {
+			return err
+		}
+
+		txBuilder.SetMemo(memo)
+
+		txFactory := tx.Factory{}
+		txFactory = txFactory.
+			WithChainID(args.chainID).
+			WithMemo(memo).
+			WithKeybase(kb).
+			WithTxConfig(clientCtx.TxConfig)
+
+		if err := tx.Sign(txFactory, nodeDirName, txBuilder, true); err != nil {
+			return err
+		}
+
+		txBz, err := clientCtx.TxConfig.TxJSONEncoder()(txBuilder.GetTx())
+		if err != nil {
+			return err
+		}
+
+		if err := writeFile(fmt.Sprintf("%v.json", nodeDirName), gentxsDir, txBz); err != nil {
+			return err
+		}
+
+		srvconfig.WriteConfigFile(filepath.Join(nodeDir, "config", "app.toml"), appConfig)
+	}
+
+	if err := initGenFiles(clientCtx, mbm, args.chainID, genAccounts, genBalances, genFiles, args.numValidators); err != nil {
+		return err
+	}
+
+	err := collectGenFiles(
+		clientCtx, nodeConfig, args.chainID, nodeIDs, valPubKeys, args.numValidators,
+		args.outputDir, args.nodeDirPrefix, args.nodeDaemonHome, genBalIterator,
+		rpcPort, p2pPortStart, args.singleMachine,
+	)
+	if err != nil {
+		return err
+	}
+
+	cmd.PrintErrf("Successfully initialized %d node directories\n", args.numValidators)
+	return nil
+}
+
+func initGenFiles(
+	clientCtx client.Context, mbm module.BasicManager, chainID string,
+	genAccounts []authtypes.GenesisAccount, genBalances []banktypes.Balance,
+	genFiles []string, numValidators int,
+) error {
+	appGenState := mbm.DefaultGenesis(clientCtx.Codec)
+
+	// set the accounts in the genesis state
+	var authGenState authtypes.GenesisState
+	clientCtx.Codec.MustUnmarshalJSON(appGenState[authtypes.ModuleName], &authGenState)
+
+	accounts, err := authtypes.PackAccounts(genAccounts)
+	if err != nil {
+		return err
+	}
+
+	authGenState.Accounts = accounts
+	appGenState[authtypes.ModuleName] = clientCtx.Codec.MustMarshalJSON(&authGenState)
+
+	// set the balances in the genesis state
+	var bankGenState banktypes.GenesisState
+	clientCtx.Codec.MustUnmarshalJSON(appGenState[banktypes.ModuleName], &bankGenState)
+
+	bankGenState.Balances = banktypes.SanitizeGenesisBalances(genBalances)
+	for _, bal := range bankGenState.Balances {
+		bankGenState.Supply = bankGenState.Supply.Add(bal.Coins...)
+	}
+	appGenState[banktypes.ModuleName] = clientCtx.Codec.MustMarshalJSON(&bankGenState)
+
+	appGenStateJSON, err := json.MarshalIndent(appGenState, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	genDoc := types.GenesisDoc{
+		ChainID:    chainID,
+		AppState:   appGenStateJSON,
+		Validators: nil,
+	}
+
+	// generate empty genesis files for each validator and save
+	for i := 0; i < numValidators; i++ {
+		if err := genDoc.SaveAs(genFiles[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func collectGenFiles(
+	clientCtx client.Context, nodeConfig *tmconfig.Config, chainID string,
+	nodeIDs []string, valPubKeys []cryptotypes.PubKey, numValidators int,
+	outputDir, nodeDirPrefix, nodeDaemonHome string, genBalIterator banktypes.GenesisBalancesIterator,
+	rpcPortStart, p2pPortStart int,
+	singleMachine bool,
+) error {
+	var appState json.RawMessage
+	genTime := tmtime.Now()
+
+	for i := 0; i < numValidators; i++ {
+		var portOffset int
+		if singleMachine {
+			portOffset = i
+		}
+
+		nodeDirName := fmt.Sprintf("%s%d", nodeDirPrefix, i)
+		nodeDir := filepath.Join(outputDir, nodeDirName, nodeDaemonHome)
+		gentxsDir := filepath.Join(outputDir, "gentxs")
+		nodeConfig.Moniker = nodeDirName
+		nodeConfig.RPC.ListenAddress = fmt.Sprintf("tcp://0.0.0.0:%d", rpcPortStart+portOffset)
+		nodeConfig.P2P.ListenAddress = fmt.Sprintf("tcp://0.0.0.0:%d", p2pPortStart+portOffset)
+
+		nodeConfig.SetRoot(nodeDir)
+
+		nodeID, valPubKey := nodeIDs[i], valPubKeys[i]
+		initCfg := genutiltypes.NewInitConfig(chainID, gentxsDir, nodeID, valPubKey)
+
+		genDoc, err := types.GenesisDocFromFile(nodeConfig.GenesisFile())
+		if err != nil {
+			return err
+		}
+
+		nodeAppState, err := genutil.GenAppStateFromConfig(clientCtx.Codec, clientCtx.TxConfig, nodeConfig, initCfg, *genDoc, genBalIterator, genutiltypes.DefaultMessageValidator)
+		if err != nil {
+			return err
+		}
+
+		if appState == nil {
+			// set the canonical application state (they should not differ)
+			appState = nodeAppState
+		}
+
+		genFile := nodeConfig.GenesisFile()
+
+		// overwrite each validator's genesis file to have a canonical genesis time
+		if err := genutil.ExportGenesisFileWithTime(genFile, chainID, nil, appState, genTime); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func getIP(i int, startingIPAddr string) (ip string, err error) {
+	if len(startingIPAddr) == 0 {
+		ip, err = server.ExternalIP()
+		if err != nil {
+			return "", err
+		}
+		return ip, nil
+	}
+	return calculateIP(startingIPAddr, i)
+}
+
+func calculateIP(ip string, i int) (string, error) {
+	ipv4 := net.ParseIP(ip).To4()
+	if ipv4 == nil {
+		return "", fmt.Errorf("%v: non ipv4 address", ip)
+	}
+
+	for j := 0; j < i; j++ {
+		ipv4[3]++
+	}
+
+	return ipv4.String(), nil
+}
+
+func writeFile(name string, dir string, contents []byte) error {
+	file := filepath.Join(dir, name)
+
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("could not create directory %q: %w", dir, err)
+	}
+
+	if err := os.WriteFile(file, contents, 0o644); err != nil { //nolint: gosec
+		return err
+	}
+
+	return nil
+}
+
+// startTestnet starts an in-process testnet
+func startTestnet(cmd *cobra.Command, args startArgs) error {
+	networkConfig := network.DefaultConfig(app.NewTestNetworkFixture)
+
+	// Default networkConfig.ChainID is random, and we should only override it if chainID provided
+	// is non-empty
+	if args.chainID != "" {
+		networkConfig.ChainID = args.chainID
+	}
+	networkConfig.SigningAlgo = args.algo
+	networkConfig.MinGasPrices = args.minGasPrices
+	networkConfig.NumValidators = args.numValidators
+	networkConfig.EnableTMLogging = args.enableLogging
+	networkConfig.RPCAddress = args.rpcAddress
+	networkConfig.APIAddress = args.apiAddress
+	networkConfig.GRPCAddress = args.grpcAddress
+	networkConfig.PrintMnemonic = args.printMnemonic
+	networkLogger := network.NewCLILogger(cmd)
+
+	baseDir := fmt.Sprintf("%s/%s", args.outputDir, networkConfig.ChainID)
+	if _, err := os.Stat(baseDir); !os.IsNotExist(err) {
+		return fmt.Errorf(
+			"testnests directory already exists for chain-id '%s': %s, please remove or select a new --chain-id",
+			networkConfig.ChainID, baseDir)
+	}
+
+	testnet, err := network.New(networkLogger, baseDir, networkConfig)
+	if err != nil {
+		return err
+	}
+
+	if _, err := testnet.WaitForHeight(1); err != nil {
+		return err
+	}
+	cmd.Println("press the Enter Key to terminate")
+	if _, err := fmt.Scanln(); err != nil { // wait for Enter Key
+		return err
+	}
+	testnet.Cleanup()
+
+	return nil
+}

tests/system/.gitignore

@@ -0,0 +1 @@
+/testnet

tests/system/README.md

@@ -0,0 +1,58 @@
+# Testing
+
+Test framework for system tests. 
+Starts and interacts with a (multi node) blockchain in Go.
+Supports
+* CLI
+* Servers
+* Events
+* RPC
+
+Uses:
+* testify
+* gjson
+* sjson
+Server and client side are executed on the host machine
+
+## Developer
+### Test strategy
+System tests cover the full stack via cli and a running (multi node) network. They are more expensive (in terms of time/ cpu) 
+to run compared to unit or integration tests. 
+Therefore, we focus on the **critical path** and do not cover every condition.
+
+### Execute a single test
+
+```sh
+go test -tags system_test -count=1 -v ./testing --run TestSmokeTest  -verbose
+```
+
+* Force a binary rebuild before running the test
+
+```sh
+go test -tags system_test -count=1 -v ./testing --run TestSmokeTest  -verbose -rebuild
+```
+
+Test cli parameters
+
+* `-verbose` verbose output
+* `-rebuild` - rebuild artifacts
+* `-wait-time` duration - time to wait for chain events (default 30s)
+* `-nodes-count` int - number of nodes in the cluster (default 4)
+
+# Port ranges
+With *n* nodes:
+* `26657` - `26657+n` - RPC
+* `1317` - `1317+n` - API
+* `9090` - `9090+n` - GRPC
+* `16656` - `16656+n` - P2P
+
+For example Node *3* listens on `26660` for RPC calls
+
+## Resources
+
+* [gjson query syntax](https://github.com/tidwall/gjson#path-syntax)
+
+## Disclaimer
+
+The initial code was contributed from the [Tgrade](https://github.com/confio/tgrade/) project. The idea was inspired by the work of the [e-money](https://github.com/e-money/em-ledger) team on their system tests. Thank
+you!

tests/system/basic_test.go

@@ -0,0 +1,124 @@
+//go:build system_test
+
+package system
+
+import (
+	"encoding/base64"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/gjson"
+)
+
+func TestBasicWasm(t *testing.T) {
+	// Scenario:
+	// upload code
+	// instantiate contract
+	// watch for an event
+	// update instantiate contract
+	// set contract admin
+	sut.ResetChain(t)
+	sut.StartChain(t)
+
+	cli := NewWasmdCLI(t, sut, verbose)
+	t.Log("List keys")
+	t.Log("keys", cli.Keys("keys", "list"))
+
+	t.Log("Upload wasm code")
+	txResult := cli.CustomCommand("tx", "wasm", "store", "./testdata/hackatom.wasm.gzip", "--from=node0", "--gas=1500000", "--fees=2stake")
+	RequireTxSuccess(t, txResult)
+
+	t.Log("Waiting for block")
+	sut.AwaitNextBlock(t)
+
+	t.Log("Query wasm code list")
+	qResult := cli.CustomQuery("q", "wasm", "list-code")
+	codes := gjson.Get(qResult, "code_infos.#.code_id").Array()
+	t.Log("got query result", qResult)
+
+	require.Equal(t, int64(1), codes[0].Int())
+	codeID := 1
+
+	l := sut.NewEventListener(t)
+	c, done := CaptureAllEventsConsumer(t)
+	expContractAddr := ContractBech32Address(1, 1)
+	query := fmt.Sprintf(`tm.event='Tx' AND wasm._contract_address='%s'`, expContractAddr)
+	t.Logf("Subscribe to events: %s", query)
+	cleanupFn := l.Subscribe(query, c)
+	t.Cleanup(cleanupFn)
+
+	t.Log("Instantiate wasm code")
+	initMsg := fmt.Sprintf(`{"verifier":%q, "beneficiary":%q}`, randomBech32Addr(), randomBech32Addr())
+	newContractAddr := cli.WasmInstantiate(codeID, initMsg, "--admin="+defaultSrcAddr, "--label=label1", "--from="+defaultSrcAddr)
+	assert.Equal(t, expContractAddr, newContractAddr)
+	assert.Len(t, done(), 1)
+
+	t.Log("Update Instantiate Config")
+	qResult = cli.CustomQuery("q", "wasm", "code-info", fmt.Sprint(codeID))
+	assert.Equal(t, "Everybody", gjson.Get(qResult, "instantiate_permission.permission").String())
+
+	rsp := cli.CustomCommand("tx", "wasm", "update-instantiate-config", fmt.Sprint(codeID), "--instantiate-anyof-addresses="+cli.GetKeyAddr(defaultSrcAddr), "--from="+defaultSrcAddr)
+	RequireTxSuccess(t, rsp)
+
+	qResult = cli.CustomQuery("q", "wasm", "code-info", fmt.Sprint(codeID))
+	t.Log(qResult)
+	assert.Equal(t, "AnyOfAddresses", gjson.Get(qResult, "instantiate_permission.permission").String())
+	assert.Equal(t, cli.GetKeyAddr(defaultSrcAddr), gjson.Get(qResult, "instantiate_permission.addresses").Array()[0].String())
+
+	t.Log("Set contract admin")
+	newAdmin := randomBech32Addr()
+	rsp = cli.CustomCommand("tx", "wasm", "set-contract-admin", newContractAddr, newAdmin, "--from="+defaultSrcAddr)
+	RequireTxSuccess(t, rsp)
+
+	qResult = cli.CustomQuery("q", "wasm", "contract", newContractAddr)
+	actualAdmin := gjson.Get(qResult, "contract_info.admin").String()
+	assert.Equal(t, newAdmin, actualAdmin)
+}
+
+func TestMultiContract(t *testing.T) {
+	// Scenario:
+	// upload reflect code
+	// upload hackatom escrow code
+	// creator instantiates a contract and gives it tokens
+	// reflect a message through the reflect to call the escrow
+	sut.ResetChain(t)
+	sut.StartChain(t)
+
+	cli := NewWasmdCLI(t, sut, verbose)
+
+	bobAddr := randomBech32Addr()
+
+	t.Log("Upload reflect code")
+	reflectID := cli.WasmStore("./testdata/reflect.wasm.gzip", "--from=node0", "--gas=1900000", "--fees=2stake")
+
+	t.Log("Upload hackatom code")
+	hackatomID := cli.WasmStore("./testdata/hackatom.wasm.gzip", "--from=node0", "--gas=1900000", "--fees=2stake")
+
+	t.Log("Instantiate reflect code")
+	reflectContractAddr := cli.WasmInstantiate(reflectID, "{}", "--admin="+defaultSrcAddr, "--label=reflect_contract", "--from="+defaultSrcAddr, "--amount=100stake")
+
+	t.Log("Instantiate hackatom code")
+	initMsg := fmt.Sprintf(`{"verifier":%q, "beneficiary":%q}`, reflectContractAddr, bobAddr)
+	hackatomContractAddr := cli.WasmInstantiate(hackatomID, initMsg, "--admin="+defaultSrcAddr, "--label=hackatom_contract", "--from="+defaultSrcAddr, "--amount=50stake")
+
+	// check balances
+	assert.Equal(t, int64(100), cli.QueryBalance(reflectContractAddr, "stake"))
+	assert.Equal(t, int64(50), cli.QueryBalance(hackatomContractAddr, "stake"))
+	assert.Equal(t, int64(0), cli.QueryBalance(bobAddr, "stake"))
+
+	// now for the trick.... we reflect a message through the reflect to call the escrow
+	// we also send an additional 20stake tokens there.
+	// this should reduce the reflect balance by 20stake (to 80stake)
+	// this 20stake is added to the escrow, then the entire balance is sent to bob (total: 70stake)
+	approveMsg := []byte(`{"release":{}}`)
+	reflectSendMsg := fmt.Sprintf(`{"reflect_msg":{"msgs":[{"wasm":{"execute":{"contract_addr":%q,"msg":%q,"funds":[{"denom":"stake","amount":"20"}]}}}]}}`, hackatomContractAddr, base64.StdEncoding.EncodeToString(approveMsg))
+	t.Log(reflectSendMsg)
+	rsp := cli.WasmExecute(reflectContractAddr, reflectSendMsg, defaultSrcAddr, "--gas=2500000", "--fees=4stake")
+	RequireTxSuccess(t, rsp)
+
+	assert.Equal(t, int64(80), cli.QueryBalance(reflectContractAddr, "stake"))
+	assert.Equal(t, int64(0), cli.QueryBalance(hackatomContractAddr, "stake"))
+	assert.Equal(t, int64(70), cli.QueryBalance(bobAddr, "stake"))
+}

tests/system/cli.go

@@ -0,0 +1,396 @@
+package system
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/cosmos/cosmos-sdk/client/rpc"
+	"github.com/cosmos/cosmos-sdk/codec"
+	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/gjson"
+	"golang.org/x/exp/slices"
+
+	"github.com/CosmWasm/wasmd/app"
+)
+
+type (
+	// blocks until next block is minted
+	awaitNextBlock func(t *testing.T, timeout ...time.Duration) int64
+	// RunErrorAssert is custom type that is satisfies by testify matchers as well
+	RunErrorAssert func(t assert.TestingT, err error, msgAndArgs ...interface{}) (ok bool)
+)
+
+// WasmdCli wraps the command line interface
+type WasmdCli struct {
+	t              *testing.T
+	nodeAddress    string
+	chainID        string
+	homeDir        string
+	fees           string
+	Debug          bool
+	amino          *codec.LegacyAmino
+	assertErrorFn  RunErrorAssert
+	awaitNextBlock awaitNextBlock
+	expTXCommitted bool
+}
+
+// NewWasmdCLI constructor
+func NewWasmdCLI(t *testing.T, sut *SystemUnderTest, verbose bool) *WasmdCli {
+	return NewWasmdCLIx(t, sut.rpcAddr, sut.chainID, sut.AwaitNextBlock, filepath.Join(workDir, sut.outputDir), "1"+sdk.DefaultBondDenom, verbose)
+}
+
+// NewWasmdCLIx extended constructor
+func NewWasmdCLIx(
+	t *testing.T,
+	nodeAddress string,
+	chainID string,
+	awaiter awaitNextBlock,
+	homeDir string,
+	fees string,
+	debug bool,
+) *WasmdCli {
+	return &WasmdCli{
+		t:              t,
+		nodeAddress:    nodeAddress,
+		chainID:        chainID,
+		homeDir:        homeDir,
+		Debug:          debug,
+		amino:          app.MakeEncodingConfig().Amino,
+		assertErrorFn:  assert.NoError,
+		awaitNextBlock: awaiter,
+		fees:           fees,
+		expTXCommitted: true,
+	}
+}
+
+// WithRunErrorsIgnored does not fail on any error
+func (c WasmdCli) WithRunErrorsIgnored() WasmdCli {
+	return c.WithRunErrorMatcher(func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool {
+		return true
+	})
+}
+
+// WithRunErrorMatcher assert function to ensure run command error value
+func (c WasmdCli) WithRunErrorMatcher(f RunErrorAssert) WasmdCli {
+	return WasmdCli{
+		t:              c.t,
+		nodeAddress:    c.nodeAddress,
+		chainID:        c.chainID,
+		homeDir:        c.homeDir,
+		Debug:          c.Debug,
+		amino:          c.amino,
+		assertErrorFn:  f,
+		awaitNextBlock: c.awaitNextBlock,
+		fees:           c.fees,
+		expTXCommitted: c.expTXCommitted,
+	}
+}
+
+func (c WasmdCli) WithNodeAddress(addr string) WasmdCli {
+	return WasmdCli{
+		t:              c.t,
+		nodeAddress:    addr,
+		chainID:        c.chainID,
+		homeDir:        c.homeDir,
+		Debug:          c.Debug,
+		amino:          c.amino,
+		assertErrorFn:  c.assertErrorFn,
+		awaitNextBlock: c.awaitNextBlock,
+		fees:           c.fees,
+		expTXCommitted: c.expTXCommitted,
+	}
+}
+
+// CustomCommand main entry for executing wasmd cli commands.
+// When configured, method blocks until tx is committed.
+func (c WasmdCli) CustomCommand(args ...string) string {
+	if c.fees != "" && !slices.ContainsFunc(args, func(s string) bool {
+		return strings.HasPrefix(s, "--fees")
+	}) {
+		args = append(args, "--fees="+c.fees) // add default fee
+	}
+	args = c.withTXFlags(args...)
+	execOutput, ok := c.run(args)
+	if !ok {
+		return execOutput
+	}
+	rsp, committed := c.awaitTxCommitted(execOutput, defaultWaitTime)
+	c.t.Logf("tx committed: %v", committed)
+	require.Equal(c.t, c.expTXCommitted, committed, "expected tx committed: %v", c.expTXCommitted)
+	return rsp
+}
+
+// wait for tx committed on chain
+func (c WasmdCli) awaitTxCommitted(submitResp string, timeout ...time.Duration) (string, bool) {
+	RequireTxSuccess(c.t, submitResp)
+	txHash := gjson.Get(submitResp, "txhash")
+	require.True(c.t, txHash.Exists())
+	var txResult string
+	for i := 0; i < 3; i++ { // max blocks to wait for a commit
+		txResult = c.WithRunErrorsIgnored().CustomQuery("q", "tx", txHash.String())
+		if code := gjson.Get(txResult, "code"); code.Exists() {
+			if code.Int() != 0 { // 0 = success code
+				c.t.Logf("+++ got error response code: %s\n", txResult)
+			}
+			return txResult, true
+		}
+		c.awaitNextBlock(c.t, timeout...)
+	}
+	return "", false
+}
+
+// Keys wasmd keys CLI command
+func (c WasmdCli) Keys(args ...string) string {
+	args = c.withKeyringFlags(args...)
+	out, _ := c.run(args)
+	return out
+}
+
+// CustomQuery main entrypoint for wasmd CLI queries
+func (c WasmdCli) CustomQuery(args ...string) string {
+	args = c.withQueryFlags(args...)
+	out, _ := c.run(args)
+	return out
+}
+
+// execute shell command
+func (c WasmdCli) run(args []string) (output string, ok bool) {
+	// todo assert error???
+	if c.Debug {
+		c.t.Logf("+++ running `wasmd %s`", strings.Join(args, " "))
+	}
+	gotOut, gotErr := func() (out []byte, err error) {
+		defer func() {
+			if r := recover(); r != nil {
+				err = fmt.Errorf("recovered from panic: %v", r)
+			}
+		}()
+		cmd := exec.Command(locateExecutable("wasmd"), args...) //nolint:gosec
+		cmd.Dir = workDir
+		return cmd.CombinedOutput()
+	}()
+	ok = c.assertErrorFn(c.t, gotErr, string(gotOut))
+	return string(gotOut), ok
+}
+
+func (c WasmdCli) withQueryFlags(args ...string) []string {
+	args = append(args, "--output", "json")
+	return c.withChainFlags(args...)
+}
+
+func (c WasmdCli) withTXFlags(args ...string) []string {
+	args = append(args,
+		"--broadcast-mode", "sync",
+		"--output", "json",
+		"--yes",
+		"--chain-id", c.chainID,
+	)
+	args = c.withKeyringFlags(args...)
+	return c.withChainFlags(args...)
+}
+
+func (c WasmdCli) withKeyringFlags(args ...string) []string {
+	r := append(args, //nolint:gocritic
+		"--home", c.homeDir,
+		"--keyring-backend", "test",
+	)
+	for _, v := range args {
+		if v == "-a" || v == "--address" { // show address only
+			return r
+		}
+	}
+	return append(r, "--output", "json")
+}
+
+func (c WasmdCli) withChainFlags(args ...string) []string {
+	return append(args,
+		"--node", c.nodeAddress,
+	)
+}
+
+// WasmExecute send MsgExecute to a contract
+func (c WasmdCli) WasmExecute(contractAddr, msg, from string, args ...string) string {
+	cmd := append([]string{"tx", "wasm", "execute", contractAddr, msg, "--from", from}, args...)
+	return c.CustomCommand(cmd...)
+}
+
+// AddKey add key to default keyring. Returns address
+func (c WasmdCli) AddKey(name string) string {
+	cmd := c.withKeyringFlags("keys", "add", name, "--no-backup")
+	out, _ := c.run(cmd)
+	addr := gjson.Get(out, "address").String()
+	require.NotEmpty(c.t, addr, "got %q", out)
+	return addr
+}
+
+// GetKeyAddr returns address
+func (c WasmdCli) GetKeyAddr(name string) string {
+	cmd := c.withKeyringFlags("keys", "show", name, "-a")
+	out, _ := c.run(cmd)
+	addr := strings.Trim(out, "\n")
+	require.NotEmpty(c.t, addr, "got %q", out)
+	return addr
+}
+
+const defaultSrcAddr = "node0"
+
+// FundAddress sends the token amount to the destination address
+func (c WasmdCli) FundAddress(destAddr, amount string) string {
+	require.NotEmpty(c.t, destAddr)
+	require.NotEmpty(c.t, amount)
+	cmd := []string{"tx", "bank", "send", defaultSrcAddr, destAddr, amount}
+	rsp := c.CustomCommand(cmd...)
+	RequireTxSuccess(c.t, rsp)
+	return rsp
+}
+
+// WasmStore uploads a wasm contract to the chain. Returns code id
+func (c WasmdCli) WasmStore(file string, args ...string) int {
+	if len(args) == 0 {
+		args = []string{"--from=" + defaultSrcAddr, "--gas=2500000"}
+	}
+	cmd := append([]string{"tx", "wasm", "store", file}, args...)
+	rsp := c.CustomCommand(cmd...)
+
+	RequireTxSuccess(c.t, rsp)
+	codeID := gjson.Get(rsp, "logs.#.events.#.attributes.#(key=code_id).value").Array()[0].Array()[0].Int()
+	require.NotEmpty(c.t, codeID)
+	return int(codeID)
+}
+
+// WasmInstantiate create a new contract instance. returns contract address
+func (c WasmdCli) WasmInstantiate(codeID int, initMsg string, args ...string) string {
+	if len(args) == 0 {
+		args = []string{"--label=testing", "--from=" + defaultSrcAddr, "--no-admin"}
+	}
+	cmd := append([]string{"tx", "wasm", "instantiate", strconv.Itoa(codeID), initMsg}, args...)
+	rsp := c.CustomCommand(cmd...)
+	RequireTxSuccess(c.t, rsp)
+	addr := gjson.Get(rsp, "logs.#.events.#.attributes.#(key=_contract_address).value").Array()[0].Array()[0].String()
+	require.NotEmpty(c.t, addr)
+	return addr
+}
+
+// QuerySmart run smart contract query
+func (c WasmdCli) QuerySmart(contractAddr, msg string, args ...string) string {
+	cmd := append([]string{"q", "wasm", "contract-state", "smart", contractAddr, msg}, args...)
+	return c.CustomQuery(cmd...)
+}
+
+// QueryBalances queries all balances for an account. Returns json response
+// Example:`{"balances":[{"denom":"node0token","amount":"1000000000"},{"denom":"stake","amount":"400000003"}],"pagination":{}}`
+func (c WasmdCli) QueryBalances(addr string) string {
+	return c.CustomQuery("q", "bank", "balances", addr)
+}
+
+// QueryBalance returns balance amount for given denom.
+// 0 when not found
+func (c WasmdCli) QueryBalance(addr, denom string) int64 {
+	raw := c.CustomQuery("q", "bank", "balances", addr, "--denom="+denom)
+	require.Contains(c.t, raw, "amount", raw)
+	return gjson.Get(raw, "amount").Int()
+}
+
+// QueryTotalSupply returns total amount of tokens for a given denom.
+// 0 when not found
+func (c WasmdCli) QueryTotalSupply(denom string) int64 {
+	raw := c.CustomQuery("q", "bank", "total", "--denom="+denom)
+	require.Contains(c.t, raw, "amount", raw)
+	return gjson.Get(raw, "amount").Int()
+}
+
+func (c WasmdCli) GetTendermintValidatorSet() rpc.ResultValidatorsOutput {
+	args := []string{"q", "tendermint-validator-set"}
+	got := c.CustomQuery(args...)
+
+	var res rpc.ResultValidatorsOutput
+	require.NoError(c.t, c.amino.UnmarshalJSON([]byte(got), &res), got)
+	return res
+}
+
+// IsInTendermintValset returns true when the giben pub key is in the current active tendermint validator set
+func (c WasmdCli) IsInTendermintValset(valPubKey cryptotypes.PubKey) (rpc.ResultValidatorsOutput, bool) {
+	valResult := c.GetTendermintValidatorSet()
+	var found bool
+	for _, v := range valResult.Validators {
+		if v.PubKey.Equals(valPubKey) {
+			found = true
+			break
+		}
+	}
+	return valResult, found
+}
+
+// RequireTxSuccess require the received response to contain the success code
+func RequireTxSuccess(t *testing.T, got string) {
+	t.Helper()
+	code, details := parseResultCode(t, got)
+	require.Equal(t, int64(0), code, "non success tx code : %s", details)
+}
+
+// RequireTxFailure require the received response to contain any failure code and the passed msgsgs
+func RequireTxFailure(t *testing.T, got string, containsMsgs ...string) {
+	t.Helper()
+	code, details := parseResultCode(t, got)
+	require.NotEqual(t, int64(0), code, details)
+	for _, msg := range containsMsgs {
+		require.Contains(t, details, msg)
+	}
+}
+
+func parseResultCode(t *testing.T, got string) (int64, string) {
+	code := gjson.Get(got, "code")
+	require.True(t, code.Exists(), "got response: %s", got)
+
+	details := got
+	if log := gjson.Get(got, "raw_log"); log.Exists() {
+		details = log.String()
+	}
+	return code.Int(), details
+}
+
+var (
+	// ErrOutOfGasMatcher requires error with "out of gas" message
+	ErrOutOfGasMatcher RunErrorAssert = func(t assert.TestingT, err error, args ...interface{}) bool {
+		const oogMsg = "out of gas"
+		return expErrWithMsg(t, err, args, oogMsg)
+	}
+	// ErrTimeoutMatcher requires time out message
+	ErrTimeoutMatcher RunErrorAssert = func(t assert.TestingT, err error, args ...interface{}) bool {
+		const expMsg = "timed out waiting for tx to be included in a block"
+		return expErrWithMsg(t, err, args, expMsg)
+	}
+	// ErrPostFailedMatcher requires post failed
+	ErrPostFailedMatcher RunErrorAssert = func(t assert.TestingT, err error, args ...interface{}) bool {
+		const expMsg = "post failed"
+		return expErrWithMsg(t, err, args, expMsg)
+	}
+	// ErrInvalidQuery requires smart query request failed
+	ErrInvalidQuery RunErrorAssert = func(t assert.TestingT, err error, args ...interface{}) bool {
+		const expMsg = "query wasm contract failed"
+		return expErrWithMsg(t, err, args, expMsg)
+	}
+)
+
+func expErrWithMsg(t assert.TestingT, err error, args []interface{}, expMsg string) bool {
+	if ok := assert.Error(t, err, args); !ok {
+		return false
+	}
+	var found bool
+	for _, v := range args {
+		if strings.Contains(fmt.Sprintf("%s", v), expMsg) {
+			found = true
+			break
+		}
+	}
+	assert.True(t, found, "expected %q but got: %s", expMsg, args)
+	return false // always abort
+}

tests/system/cli_test.go

@@ -0,0 +1,111 @@
+//go:build system_test
+
+package system
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/gjson"
+)
+
+func TestUnsafeResetAll(t *testing.T) {
+	// scenario:
+	// 	given a non-empty wasm dir exists in the node home
+	//  when `unsafe-reset-all` is executed
+	// 	then the dir and all files in it are removed
+
+	wasmDir := filepath.Join(workDir, sut.nodePath(0), "wasm")
+	require.NoError(t, os.MkdirAll(wasmDir, os.ModePerm))
+
+	_, err := os.CreateTemp(wasmDir, "testing")
+	require.NoError(t, err)
+
+	// when
+	sut.ForEachNodeExecAndWait(t, []string{"tendermint", "unsafe-reset-all"})
+
+	// then
+	sut.withEachNodeHome(func(i int, home string) {
+		if _, err := os.Stat(wasmDir); !os.IsNotExist(err) {
+			t.Fatal("expected wasm dir to be removed")
+		}
+	})
+}
+
+func TestVestingAccounts(t *testing.T) {
+	// Scenario:
+	//   given: a genesis file
+	//   when: add-genesis-account with vesting flags is executed
+	//   then: the vesting account data is added to the genesis
+	sut.ResetChain(t)
+	cli := NewWasmdCLI(t, sut, verbose)
+	vest1Addr := cli.AddKey("vesting1")
+	vest2Addr := cli.AddKey("vesting2")
+	vest3Addr := cli.AddKey("vesting3")
+	myStartTimestamp := time.Now().Add(time.Minute).Unix()
+	myEndTimestamp := time.Now().Add(time.Hour).Unix()
+	sut.ModifyGenesisCLI(t,
+		// delayed vesting no cash
+		[]string{"genesis", "add-genesis-account", vest1Addr, "100000000stake", "--vesting-amount=100000000stake", fmt.Sprintf("--vesting-end-time=%d", myEndTimestamp)},
+		// continuous vesting no cash
+		[]string{"genesis", "add-genesis-account", vest2Addr, "100000001stake", "--vesting-amount=100000001stake", fmt.Sprintf("--vesting-start-time=%d", myStartTimestamp), fmt.Sprintf("--vesting-end-time=%d", myEndTimestamp)},
+		// continuous vesting with some cash
+		[]string{"genesis", "add-genesis-account", vest3Addr, "200000002stake", "--vesting-amount=100000002stake", fmt.Sprintf("--vesting-start-time=%d", myStartTimestamp), fmt.Sprintf("--vesting-end-time=%d", myEndTimestamp)},
+	)
+	raw := sut.ReadGenesisJSON(t)
+	// delayed vesting: without a start time
+	accounts := gjson.GetBytes([]byte(raw), `app_state.auth.accounts.#[@type=="/cosmos.vesting.v1beta1.DelayedVestingAccount"]#`).Array()
+	require.Len(t, accounts, 1)
+	gotAddr := accounts[0].Get("base_vesting_account.base_account.address").String()
+	assert.Equal(t, vest1Addr, gotAddr)
+	amounts := accounts[0].Get("base_vesting_account.original_vesting").Array()
+	require.Len(t, amounts, 1)
+	assert.Equal(t, "stake", amounts[0].Get("denom").String())
+	assert.Equal(t, "100000000", amounts[0].Get("amount").String())
+	assert.Equal(t, myEndTimestamp, accounts[0].Get("base_vesting_account.end_time").Int())
+	assert.Equal(t, int64(0), accounts[0].Get("start_time").Int())
+
+	// continuous vesting: start time
+	accounts = gjson.GetBytes([]byte(raw), `app_state.auth.accounts.#[@type=="/cosmos.vesting.v1beta1.ContinuousVestingAccount"]#`).Array()
+	require.Len(t, accounts, 2)
+	gotAddr = accounts[0].Get("base_vesting_account.base_account.address").String()
+	assert.Equal(t, vest2Addr, gotAddr)
+	amounts = accounts[0].Get("base_vesting_account.original_vesting").Array()
+	require.Len(t, amounts, 1)
+	assert.Equal(t, "stake", amounts[0].Get("denom").String())
+	assert.Equal(t, "100000001", amounts[0].Get("amount").String())
+	assert.Equal(t, myEndTimestamp, accounts[0].Get("base_vesting_account.end_time").Int())
+	assert.Equal(t, myStartTimestamp, accounts[0].Get("start_time").Int())
+	// with some cash
+	gotAddr = accounts[1].Get("base_vesting_account.base_account.address").String()
+	assert.Equal(t, vest3Addr, gotAddr)
+	amounts = accounts[1].Get("base_vesting_account.original_vesting").Array()
+	require.Len(t, amounts, 1)
+	assert.Equal(t, "stake", amounts[0].Get("denom").String())
+	assert.Equal(t, "100000002", amounts[0].Get("amount").String())
+	assert.Equal(t, myEndTimestamp, accounts[0].Get("base_vesting_account.end_time").Int())
+	assert.Equal(t, myStartTimestamp, accounts[0].Get("start_time").Int())
+
+	// check accounts have some balances
+	assert.Equal(t, sdk.NewCoins(sdk.NewCoin("stake", sdk.NewInt(100000000))), getGenesisBalance([]byte(raw), vest1Addr))
+	assert.Equal(t, sdk.NewCoins(sdk.NewCoin("stake", sdk.NewInt(100000001))), getGenesisBalance([]byte(raw), vest2Addr))
+	assert.Equal(t, sdk.NewCoins(sdk.NewCoin("stake", sdk.NewInt(200000002))), getGenesisBalance([]byte(raw), vest3Addr))
+}
+
+func getGenesisBalance(raw []byte, addr string) sdk.Coins {
+	var r []sdk.Coin
+	balances := gjson.GetBytes(raw, fmt.Sprintf(`app_state.bank.balances.#[address==%q]#.coins`, addr)).Array()
+	for _, coins := range balances {
+		for _, coin := range coins.Array() {
+			r = append(r, sdk.NewCoin(coin.Get("denom").String(), sdk.NewInt(coin.Get("amount").Int())))
+		}
+	}
+	return r
+}

tests/system/fraud_test.go

@@ -0,0 +1,79 @@
+//go:build system_test
+
+package system
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	sdkmath "cosmossdk.io/math"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecursiveMsgsExternalTrigger(t *testing.T) {
+	const maxBlockGas = 2_000_000
+	sut.ModifyGenesisJSON(t, SetConsensusMaxGas(t, maxBlockGas))
+	sut.StartChain(t)
+	cli := NewWasmdCLI(t, sut, verbose)
+
+	codeID := cli.WasmStore("./testdata/hackatom.wasm.gzip", "--from=node0", "--gas=1500000", "--fees=2stake")
+	initMsg := fmt.Sprintf(`{"verifier":%q, "beneficiary":%q}`, randomBech32Addr(), randomBech32Addr())
+	contractAddr := cli.WasmInstantiate(codeID, initMsg)
+
+	specs := map[string]struct {
+		gas           string
+		expErrMatcher RunErrorAssert
+	}{
+		"simulation": {
+			gas:           "auto",
+			expErrMatcher: ErrOutOfGasMatcher,
+		},
+	}
+	for name, spec := range specs {
+		t.Run(name, func(t *testing.T) {
+			cli := NewWasmdCLI(t, sut, verbose)
+			execMsg := `{"message_loop":{}}`
+			fees := "1stake"
+			gas := spec.gas
+			if gas != "auto" {
+				fees = calcMinFeeRequired(t, gas)
+			}
+			for _, n := range sut.AllNodes(t) {
+				clix := cli.WithRunErrorMatcher(spec.expErrMatcher).WithNodeAddress(n.RPCAddr())
+				clix.expTXCommitted = false
+				clix.WasmExecute(contractAddr, execMsg, defaultSrcAddr, "--gas="+gas, "--broadcast-mode=sync", "--fees="+fees)
+			}
+			sut.AwaitNextBlock(t)
+		})
+	}
+}
+
+func TestRecursiveSmartQuery(t *testing.T) {
+	sut.ResetDirtyChain(t)
+	sut.StartChain(t)
+	cli := NewWasmdCLI(t, sut, verbose)
+
+	initMsg := fmt.Sprintf(`{"verifier":%q, "beneficiary":%q}`, randomBech32Addr(), randomBech32Addr())
+	maliciousContractAddr := cli.WasmInstantiate(cli.WasmStore("./testdata/hackatom.wasm.gzip", "--from=node0", "--gas=1500000", "--fees=2stake"), initMsg)
+
+	msg := fmt.Sprintf(`{"recurse":{"depth":%d, "work":0}}`, math.MaxUint32)
+
+	// when
+	for _, n := range sut.AllNodes(t) {
+		cli.WithRunErrorMatcher(ErrInvalidQuery).WithNodeAddress(n.RPCAddr()).
+			QuerySmart(maliciousContractAddr, msg)
+	}
+	sut.AwaitNextBlock(t)
+}
+
+// with default gas factor and token
+func calcMinFeeRequired(t *testing.T, gas string) string {
+	x, ok := sdkmath.NewIntFromString(gas)
+	require.True(t, ok)
+	const defaultTestnetFee = "0.000006"
+	minFee, err := sdkmath.LegacyNewDecFromStr(defaultTestnetFee)
+	require.NoError(t, err)
+	return fmt.Sprintf("%sstake", minFee.Mul(sdkmath.LegacyNewDecFromInt(x)).RoundInt().String())
+}

tests/system/genesis_mutators.go

@@ -0,0 +1,19 @@
+package system
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/sjson"
+)
+
+// SetConsensusMaxGas max gas that can be consumed in a block
+func SetConsensusMaxGas(t *testing.T, max int) GenesisMutator {
+	return func(genesis []byte) []byte {
+		t.Helper()
+		state, err := sjson.SetRawBytes(genesis, "consensus_params.block.max_gas", []byte(fmt.Sprintf(`"%d"`, max)))
+		require.NoError(t, err)
+		return state
+	}
+}

tests/system/go.mod

@@ -0,0 +1,186 @@
+module github.com/CosmWasm/wasmd/tests/system
+
+go 1.19
+
+require (
+	github.com/CosmWasm/wasmd v0.40.0-rc.2
+	github.com/CosmWasm/wasmvm v1.2.3 // indirect
+	github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect
+	github.com/cosmos/cosmos-sdk v0.47.2
+	github.com/cosmos/gogogateway v1.2.0 // indirect
+	github.com/cosmos/gogoproto v1.4.8 // indirect
+	github.com/cosmos/iavl v0.20.0 // indirect
+	github.com/cosmos/ibc-go/v7 v7.0.0 // indirect
+	github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.15.0 // indirect
+	github.com/rakyll/statik v0.1.7 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	github.com/spf13/cobra v1.6.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.2
+	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
+	google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44 // indirect
+	google.golang.org/grpc v1.54.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)
+
+require (
+	github.com/cometbft/cometbft v0.37.1
+	github.com/tidwall/gjson v1.14.2
+	github.com/tidwall/sjson v1.2.5
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
+)
+
+require (
+	cloud.google.com/go v0.110.0 // indirect
+	cloud.google.com/go/compute v1.18.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v0.12.0 // indirect
+	cloud.google.com/go/storage v1.29.0 // indirect
+	cosmossdk.io/api v0.3.1 // indirect
+	cosmossdk.io/core v0.5.1 // indirect
+	cosmossdk.io/depinject v1.0.0-alpha.3 // indirect
+	cosmossdk.io/errors v1.0.0-beta.7 // indirect
+	cosmossdk.io/math v1.0.1 // indirect
+	cosmossdk.io/tools/rosetta v0.2.1 // indirect
+	filippo.io/edwards25519 v1.0.0 // indirect
+	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
+	github.com/99designs/keyring v1.2.1 // indirect
+	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/aws/aws-sdk-go v1.44.203 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
+	github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chzyer/readline v1.5.1 // indirect
+	github.com/cockroachdb/apd/v2 v2.0.2 // indirect
+	github.com/coinbase/rosetta-sdk-go/types v1.0.0 // indirect
+	github.com/cometbft/cometbft-db v0.7.0 // indirect
+	github.com/confio/ics23/go v0.9.0 // indirect
+	github.com/cosmos/btcutil v1.0.5 // indirect
+	github.com/cosmos/go-bip39 v1.0.0 // indirect
+	github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect
+	github.com/cosmos/rosetta-sdk-go v0.10.0 // indirect
+	github.com/creachadair/taskgroup v0.4.2 // indirect
+	github.com/danieljoos/wincred v1.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
+	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/felixge/httpsnoop v1.0.2 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-kit/kit v0.12.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
+	github.com/gogo/googleapis v1.4.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.1.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.1.2 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/orderedcode v0.0.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
+	github.com/gorilla/handlers v1.5.1 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
+	github.com/gtank/merlin v0.1.1 // indirect
+	github.com/gtank/ristretto255 v0.1.2 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-getter v1.7.1 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hdevalence/ed25519consensus v0.1.0 // indirect
+	github.com/huandu/skiplist v1.2.0 // indirect
+	github.com/improbable-eng/grpc-web v0.15.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jmhodges/levigo v1.0.0 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/lib/pq v1.10.7 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/manifoldco/promptui v0.9.0 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
+	github.com/minio/highwayhash v1.0.2 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mtibben/percent v0.2.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
+	github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rs/cors v1.8.3 // indirect
+	github.com/sasha-s/go-deadlock v0.3.1 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/viper v1.15.0 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
+	github.com/tendermint/go-amino v0.16.0 // indirect
+	github.com/tidwall/btree v1.6.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/ulikunitz/xz v0.5.11 // indirect
+	github.com/zondax/hid v0.9.1 // indirect
+	github.com/zondax/ledger-go v0.14.1 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/term v0.7.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/api v0.110.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	nhooyr.io/websocket v1.8.6 // indirect
+	pgregory.net/rapid v0.5.5 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace (
+	github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0
+	// dgrijalva/jwt-go is deprecated and doesn't receive security updates.
+	// See: https://github.com/cosmos/cosmos-sdk/issues/13134
+	github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2
+	// Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
+	// See: https://github.com/cosmos/cosmos-sdk/issues/10409
+	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.8.1
+
+	// pin version! 126854af5e6d has issues with the store so that queries fail
+	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+)

tests/system/main_test.go

@@ -0,0 +1,138 @@
+//go:build system_test
+
+package system
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/cometbft/cometbft/libs/rand"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/types/address"
+	"github.com/stretchr/testify/require"
+
+	"github.com/CosmWasm/wasmd/app"
+	wasmkeeper "github.com/CosmWasm/wasmd/x/wasm/keeper"
+)
+
+var (
+	sut     *SystemUnderTest
+	verbose bool
+)
+
+func init() {
+	config := sdk.GetConfig()
+	config.SetBech32PrefixForAccount(app.Bech32PrefixAccAddr, app.Bech32PrefixAccPub)
+	config.SetBech32PrefixForValidator(app.Bech32PrefixValAddr, app.Bech32PrefixValPub)
+	config.SetBech32PrefixForConsensusNode(app.Bech32PrefixConsAddr, app.Bech32PrefixConsPub)
+}
+
+func TestMain(m *testing.M) {
+	rebuild := flag.Bool("rebuild", false, "rebuild artifacts")
+	waitTime := flag.Duration("wait-time", defaultWaitTime, "time to wait for chain events")
+	nodesCount := flag.Int("nodes-count", 4, "number of nodes in the cluster")
+	blockTime := flag.Duration("block-time", 1000*time.Millisecond, "block creation time")
+	flag.BoolVar(&verbose, "verbose", false, "verbose output")
+	flag.Parse()
+
+	// fail fast on most common setup issue
+	requireEnoughFileHandlers(*nodesCount + 1) // +1 as tests may start another node
+
+	dir, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	workDir = dir
+	if verbose {
+		println("Work dir: ", workDir)
+	}
+	defaultWaitTime = *waitTime
+	sut = NewSystemUnderTest(verbose, *nodesCount, *blockTime)
+	if *rebuild {
+		sut.BuildNewBinary()
+	}
+	// setup chain and keyring
+	sut.SetupChain()
+
+	// run tests
+	exitCode := m.Run()
+
+	// postprocess
+	sut.StopChain()
+	if verbose || exitCode != 0 {
+		sut.PrintBuffer()
+		printResultFlag(exitCode == 0)
+	}
+
+	os.Exit(exitCode)
+}
+
+// requireEnoughFileHandlers uses `ulimit`
+func requireEnoughFileHandlers(nodesCount int) error {
+	ulimit, err := exec.LookPath("ulimit")
+	if err != nil || ulimit == "" { // skip when not available
+		return nil
+	}
+
+	cmd := exec.Command(ulimit, "-n")
+	cmd.Dir = workDir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error :%#+v, output: %s", err, string(out)))
+	}
+	fileDescrCount, err := strconv.Atoi(strings.Trim(string(out), " \t\n"))
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error :%#+v, output: %s", err, string(out)))
+	}
+	expFH := nodesCount * 260 // random number that worked on my box
+	if fileDescrCount < expFH {
+		panic(fmt.Sprintf("Fail fast. Insufficient setup. Run 'ulimit -n %d'", expFH))
+	}
+	return err
+}
+
+const (
+	successFlag = `
+ ___ _   _  ___ ___ ___  ___ ___ 
+/ __| | | |/ __/ __/ _ \/ __/ __|
+\__ \ |_| | (_| (_|  __/\__ \__ \
+|___/\__,_|\___\___\___||___/___/`
+	failureFlag = `
+  __      _ _          _ 
+ / _|    (_) |        | |
+| |_ __ _ _| | ___  __| |
+|  _/ _| | | |/ _ \/ _| |
+| || (_| | | |  __/ (_| |
+|_| \__,_|_|_|\___|\__,_|`
+)
+
+func printResultFlag(ok bool) {
+	if ok {
+		fmt.Println(successFlag)
+	} else {
+		fmt.Println(failureFlag)
+	}
+}
+
+func randomBech32Addr() string {
+	src := rand.Bytes(address.Len)
+	return sdk.AccAddress(src).String()
+}
+
+// ContractBech32Address build a wasmd bech32 contract address
+func ContractBech32Address(codeID, instanceID uint64) string {
+	return wasmkeeper.BuildContractAddressClassic(codeID, instanceID).String()
+}
+
+func toJson(t *testing.T, o interface{}) string {
+	bz, err := json.Marshal(o)
+	require.NoError(t, err)
+	return string(bz)
+}

tests/system/rpc_client.go

@@ -0,0 +1,32 @@
+package system
+
+import (
+	"context"
+	"testing"
+
+	client "github.com/cometbft/cometbft/rpc/client/http"
+	cmtypes "github.com/cometbft/cometbft/types"
+	"github.com/stretchr/testify/require"
+)
+
+// RPCClient is a test helper to interact with a node via the RPC endpoint.
+type RPCClient struct {
+	client *client.HTTP
+	t      *testing.T
+}
+
+// NewRPCClient constructor
+func NewRPCClient(t *testing.T, addr string) RPCClient {
+	httpClient, err := client.New(addr, "/websocket")
+	require.NoError(t, err)
+	require.NoError(t, httpClient.Start())
+	t.Cleanup(func() { _ = httpClient.Stop() })
+	return RPCClient{client: httpClient, t: t}
+}
+
+// Validators returns list of validators
+func (r RPCClient) Validators() []*cmtypes.Validator {
+	v, err := r.client.Validators(context.Background(), nil, nil, nil)
+	require.NoError(r.t, err)
+	return v.Validators
+}

tests/system/staking_test.go

@@ -0,0 +1,54 @@
+//go:build system_test
+
+package system
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
+)
+
+func TestStakeUnstake(t *testing.T) {
+	// Scenario:
+	// delegate tokens to validator
+	// undelegate some tokens
+
+	sut.ResetChain(t)
+
+	cli := NewWasmdCLI(t, sut, verbose)
+
+	// add genesis account with some tokens
+	account1Addr := cli.AddKey("account1")
+	sut.ModifyGenesisCLI(t,
+		[]string{"genesis", "add-genesis-account", account1Addr, "100000000stake"},
+	)
+
+	sut.StartChain(t)
+
+	// query validator address to delegate tokens
+	rsp := cli.CustomQuery("q", "staking", "validators")
+	valAddr := gjson.Get(rsp, "validators.#.operator_address").Array()[0].String()
+
+	// stake tokens
+	rsp = cli.CustomCommand("tx", "staking", "delegate", valAddr, "10000stake", "--from="+account1Addr, "--fees=1stake")
+	RequireTxSuccess(t, rsp)
+
+	t.Log(cli.QueryBalance(account1Addr, "stake"))
+	assert.Equal(t, int64(99989999), cli.QueryBalance(account1Addr, "stake"))
+
+	rsp = cli.CustomQuery("q", "staking", "delegation", account1Addr, valAddr)
+	assert.Equal(t, "10000", gjson.Get(rsp, "balance.amount").String())
+	assert.Equal(t, "stake", gjson.Get(rsp, "balance.denom").String())
+
+	// unstake tokens
+	rsp = cli.CustomCommand("tx", "staking", "unbond", valAddr, "5000stake", "--from="+account1Addr, "--fees=1stake")
+	RequireTxSuccess(t, rsp)
+
+	rsp = cli.CustomQuery("q", "staking", "delegation", account1Addr, valAddr)
+	assert.Equal(t, "5000", gjson.Get(rsp, "balance.amount").String())
+	assert.Equal(t, "stake", gjson.Get(rsp, "balance.denom").String())
+
+	rsp = cli.CustomQuery("q", "staking", "unbonding-delegation", account1Addr, valAddr)
+	assert.Equal(t, "5000", gjson.Get(rsp, "entries.#.balance").Array()[0].String())
+}

tests/system/system.go

@@ -0,0 +1,818 @@
+package system
+
+import (
+	"bufio"
+	"bytes"
+	"container/ring"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+
+	"github.com/tidwall/sjson"
+
+	"github.com/cometbft/cometbft/libs/sync"
+	client "github.com/cometbft/cometbft/rpc/client/http"
+	ctypes "github.com/cometbft/cometbft/rpc/core/types"
+	tmtypes "github.com/cometbft/cometbft/types"
+	"github.com/cosmos/cosmos-sdk/server"
+	"github.com/stretchr/testify/require"
+)
+
+var workDir string
+
+// SystemUnderTest blockchain provisioning
+type SystemUnderTest struct {
+	blockListener     *EventListener
+	currentHeight     int64
+	chainID           string
+	outputDir         string
+	blockTime         time.Duration
+	rpcAddr           string
+	initialNodesCount int
+	nodesCount        int
+	minGasPrice       string
+	cleanupFn         []CleanupFn
+	outBuff           *ring.Ring
+	errBuff           *ring.Ring
+	out               io.Writer
+	verbose           bool
+	ChainStarted      bool
+	dirty             bool // requires full reset when marked dirty
+}
+
+func NewSystemUnderTest(verbose bool, nodesCount int, blockTime time.Duration) *SystemUnderTest {
+	return &SystemUnderTest{
+		chainID:           "testing",
+		outputDir:         "./testnet",
+		blockTime:         blockTime,
+		rpcAddr:           "tcp://localhost:26657",
+		initialNodesCount: nodesCount,
+		outBuff:           ring.New(100),
+		errBuff:           ring.New(100),
+		out:               os.Stdout,
+		verbose:           verbose,
+		minGasPrice:       fmt.Sprintf("0.000001%s", sdk.DefaultBondDenom),
+	}
+}
+
+func (s *SystemUnderTest) SetupChain() {
+	s.Logf("Setup chain: %s\n", s.outputDir)
+	if err := os.RemoveAll(filepath.Join(workDir, s.outputDir)); err != nil {
+		panic(err.Error())
+	}
+
+	args := []string{
+		"testnet",
+		"init-files",
+		"--chain-id=" + s.chainID,
+		"--output-dir=" + s.outputDir,
+		"--v=" + strconv.Itoa(s.initialNodesCount),
+		"--keyring-backend=test",
+		"--commit-timeout=" + s.blockTime.String(),
+		"--minimum-gas-prices=" + s.minGasPrice,
+		"--starting-ip-address", "", // empty to use host systems
+		"--single-host",
+	}
+	println("+++ wasmd " + strings.Join(args, " "))
+	cmd := exec.Command( //nolint:gosec
+		locateExecutable("wasmd"),
+		args...,
+	)
+	cmd.Dir = workDir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error :%#+v, output: %s", err, string(out)))
+	}
+	s.Log(string(out))
+	s.nodesCount = s.initialNodesCount
+
+	// modify genesis with system test defaults
+	src := filepath.Join(workDir, s.nodePath(0), "config", "genesis.json")
+	genesisBz, err := ioutil.ReadFile(src)
+	if err != nil {
+		panic(fmt.Sprintf("failed to load genesis: %s", err))
+	}
+
+	genesisBz, err = sjson.SetRawBytes(genesisBz, "consensus_params.block.max_gas", []byte(fmt.Sprintf(`"%d"`, 10_000_000)))
+	if err != nil {
+		panic(fmt.Sprintf("failed set block max gas: %s", err))
+	}
+	s.withEachNodeHome(func(i int, home string) {
+		if err := saveGenesis(home, genesisBz); err != nil {
+			panic(err)
+		}
+	})
+
+	// backup genesis
+	dest := filepath.Join(workDir, s.nodePath(0), "config", "genesis.json.orig")
+	if _, err := copyFile(src, dest); err != nil {
+		panic(fmt.Sprintf("copy failed :%#+v", err))
+	}
+	// backup keyring
+	src = filepath.Join(workDir, s.nodePath(0), "keyring-test")
+	dest = filepath.Join(workDir, s.outputDir, "keyring-test")
+	if err := copyFilesInDir(src, dest); err != nil {
+		panic(fmt.Sprintf("copy files from dir :%#+v", err))
+	}
+}
+
+func (s *SystemUnderTest) StartChain(t *testing.T, xargs ...string) {
+	t.Helper()
+	s.Log("Start chain\n")
+	s.ChainStarted = true
+	s.forEachNodesExecAsync(t, append([]string{"start", "--trace", "--log_level=info"}, xargs...)...)
+
+	s.AwaitNodeUp(t, s.rpcAddr)
+
+	t.Log("Start new block listener")
+	s.blockListener = NewEventListener(t, s.rpcAddr)
+	s.cleanupFn = append(s.cleanupFn,
+		s.blockListener.Subscribe("tm.event='NewBlock'", func(e ctypes.ResultEvent) (more bool) {
+			newBlock, ok := e.Data.(tmtypes.EventDataNewBlock)
+			require.True(t, ok, "unexpected type %T", e.Data)
+			atomic.StoreInt64(&s.currentHeight, newBlock.Block.Height)
+			return true
+		}),
+	)
+	s.AwaitNextBlock(t, 4e9)
+}
+
+// MarkDirty whole chain will be reset when marked dirty
+func (s *SystemUnderTest) MarkDirty() {
+	s.dirty = true
+}
+
+// IsDirty true when non default genesis or other state modification were applied that might create incompatibility for tests
+func (s SystemUnderTest) IsDirty() bool {
+	return s.dirty
+}
+
+// watchLogs stores stdout/stderr in a file and in a ring buffer to output the last n lines on test error
+func (s *SystemUnderTest) watchLogs(node int, cmd *exec.Cmd) {
+	logfile, err := os.Create(filepath.Join(workDir, s.outputDir, fmt.Sprintf("node%d.out", node)))
+	if err != nil {
+		panic(fmt.Sprintf("open logfile error %#+v", err))
+	}
+
+	errReader, err := cmd.StderrPipe()
+	if err != nil {
+		panic(fmt.Sprintf("stderr reader error %#+v", err))
+	}
+	stopRingBuffer := make(chan struct{})
+	go appendToBuf(io.TeeReader(errReader, logfile), s.errBuff, stopRingBuffer)
+
+	outReader, err := cmd.StdoutPipe()
+	if err != nil {
+		panic(fmt.Sprintf("stdout reader error %#+v", err))
+	}
+	go appendToBuf(io.TeeReader(outReader, logfile), s.outBuff, stopRingBuffer)
+	s.cleanupFn = append(s.cleanupFn, func() {
+		close(stopRingBuffer)
+		logfile.Close()
+	})
+}
+
+func appendToBuf(r io.Reader, b *ring.Ring, stop <-chan struct{}) {
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		select {
+		case <-stop:
+			return
+		default:
+		}
+		text := scanner.Text()
+		// filter out noise
+		if strings.Contains(text, "module=rpc-server protocol=websocket") {
+			continue
+		}
+		b.Value = text
+		b = b.Next()
+	}
+}
+
+// AwaitNodeUp ensures the node is running
+func (s *SystemUnderTest) AwaitNodeUp(t *testing.T, rpcAddr string) {
+	t.Helper()
+	t.Logf("Await node is up: %s", rpcAddr)
+	timeout := defaultWaitTime
+	ctx, done := context.WithTimeout(context.Background(), timeout)
+	defer done()
+
+	started := make(chan struct{})
+	go func() { // query for a non empty block on status page
+		t.Logf("Checking node status: %s\n", rpcAddr)
+		for {
+			con, err := client.New(rpcAddr, "/websocket")
+			if err != nil || con.Start() != nil {
+				time.Sleep(time.Second)
+				continue
+			}
+			result, err := con.Status(ctx)
+			if err != nil || result.SyncInfo.LatestBlockHeight < 1 {
+				con.Stop() //nolint:errcheck
+				continue
+			}
+			t.Logf("Node started. Current block: %d\n", result.SyncInfo.LatestBlockHeight)
+			con.Stop() //nolint:errcheck
+			started <- struct{}{}
+		}
+	}()
+	select {
+	case <-started:
+	case <-ctx.Done():
+		require.NoError(t, ctx.Err())
+	case <-time.NewTimer(timeout).C:
+		t.Fatalf("timeout waiting for node start: %s", timeout)
+	}
+}
+
+// StopChain stops the system under test and executes all registered cleanup callbacks
+func (s *SystemUnderTest) StopChain() {
+	s.Log("Stop chain\n")
+	if !s.ChainStarted {
+		return
+	}
+
+	for _, c := range s.cleanupFn {
+		c()
+	}
+	s.cleanupFn = nil
+	// send SIGTERM
+	cmd := exec.Command(locateExecutable("pkill"), "-15", "wasmd") //nolint:gosec
+	cmd.Dir = workDir
+	if _, err := cmd.CombinedOutput(); err != nil {
+		s.Logf("failed to stop chain: %s\n", err)
+	}
+
+	var shutdown bool
+	for timeout := time.NewTimer(500 * time.Millisecond).C; !shutdown; {
+		select {
+		case <-timeout:
+			s.Log("killing nodes now")
+			cmd = exec.Command(locateExecutable("pkill"), "-9", "wasmd") //nolint:gosec
+			cmd.Dir = workDir
+			if _, err := cmd.CombinedOutput(); err != nil {
+				s.Logf("failed to kill process: %s\n", err)
+			}
+			shutdown = true
+		default:
+			if err := exec.Command(locateExecutable("pgrep"), "wasmd").Run(); err != nil { //nolint:gosec
+				shutdown = true
+			}
+		}
+	}
+	s.ChainStarted = false
+}
+
+// PrintBuffer prints the chain logs to the console
+func (s SystemUnderTest) PrintBuffer() {
+	s.outBuff.Do(func(v interface{}) {
+		if v != nil {
+			fmt.Fprintf(s.out, "out> %s\n", v)
+		}
+	})
+	fmt.Fprint(s.out, "8< chain err -----------------------------------------\n")
+	s.errBuff.Do(func(v interface{}) {
+		if v != nil {
+			fmt.Fprintf(s.out, "err> %s\n", v)
+		}
+	})
+}
+
+// BuildNewBinary builds and installs new wasmd binary
+func (s SystemUnderTest) BuildNewBinary() {
+	s.Log("Install binaries\n")
+	makePath := locateExecutable("make")
+	cmd := exec.Command(makePath, "clean", "install")
+	cmd.Dir = workDir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error %#v : output: %s", err, string(out)))
+	}
+}
+
+// AwaitNextBlock is a first class function that any caller can use to ensure a new block was minted.
+// Returns the new height
+func (s *SystemUnderTest) AwaitNextBlock(t *testing.T, timeout ...time.Duration) int64 {
+	t.Helper()
+	maxWaitTime := s.blockTime * 3
+	if len(timeout) != 0 { // optional argument to overwrite default timeout
+		maxWaitTime = timeout[0]
+	}
+	done := make(chan int64)
+	go func() {
+		for start, current := atomic.LoadInt64(&s.currentHeight), atomic.LoadInt64(&s.currentHeight); current == start; current = atomic.LoadInt64(&s.currentHeight) {
+			time.Sleep(s.blockTime)
+		}
+		done <- atomic.LoadInt64(&s.currentHeight)
+		close(done)
+	}()
+	select {
+	case v := <-done:
+		return v
+	case <-time.NewTimer(maxWaitTime).C:
+		t.Fatalf("Timeout - no block within %s", maxWaitTime)
+		return -1
+	}
+}
+
+// ResetDirtyChain reset chain when non default setup or state (dirty)
+func (s *SystemUnderTest) ResetDirtyChain(t *testing.T) {
+	if s.IsDirty() {
+		s.ResetChain(t)
+	}
+}
+
+// ResetChain stops and clears all nodes state via 'unsafe-reset-all'
+func (s *SystemUnderTest) ResetChain(t *testing.T) {
+	t.Helper()
+	t.Log("Reset chain")
+	s.StopChain()
+	restoreOriginalGenesis(t, *s)
+	restoreOriginalKeyring(t, *s)
+	s.resetBuffers()
+
+	// remove all additional nodes
+	for i := s.initialNodesCount; i < s.nodesCount; i++ {
+		os.RemoveAll(filepath.Join(workDir, s.nodePath(i)))
+		os.Remove(filepath.Join(workDir, s.outputDir, fmt.Sprintf("node%d.out", i)))
+	}
+	s.nodesCount = s.initialNodesCount
+
+	// reset all validataor nodes
+	s.ForEachNodeExecAndWait(t, []string{"tendermint", "unsafe-reset-all"})
+	s.currentHeight = 0
+	s.dirty = false
+}
+
+// ModifyGenesisCLI executes the CLI commands to modify the genesis
+func (s *SystemUnderTest) ModifyGenesisCLI(t *testing.T, cmds ...[]string) {
+	s.ForEachNodeExecAndWait(t, cmds...)
+	s.MarkDirty()
+}
+
+type GenesisMutator func([]byte) []byte
+
+// ModifyGenesisJSON resets the chain and executes the callbacks to update the json representation
+// The mutator callbacks after each other receive the genesis as raw bytes and return the updated genesis for the next.
+// example:
+//
+//	return func(genesis []byte) []byte {
+//		val, _ := json.Marshal(sdk.NewDecCoins(fees...))
+//		state, _ := sjson.SetRawBytes(genesis, "app_state.globalfee.params.minimum_gas_prices", val)
+//		return state
+//	}
+func (s *SystemUnderTest) ModifyGenesisJSON(t *testing.T, mutators ...GenesisMutator) {
+	s.ResetChain(t)
+	s.modifyGenesisJSON(t, mutators...)
+}
+
+// modify json without enforcing a reset
+func (s *SystemUnderTest) modifyGenesisJSON(t *testing.T, mutators ...GenesisMutator) {
+	require.Empty(t, s.currentHeight, "forced chain reset required")
+	current, err := os.ReadFile(filepath.Join(workDir, s.nodePath(0), "config", "genesis.json"))
+	require.NoError(t, err)
+	for _, m := range mutators {
+		current = m(current)
+	}
+	out := storeTempFile(t, current)
+	defer os.Remove(out.Name())
+	s.setGenesis(t, out.Name())
+	s.MarkDirty()
+}
+
+// ReadGenesisJSON returns current genesis.json content as raw string
+func (s *SystemUnderTest) ReadGenesisJSON(t *testing.T) string {
+	content, err := os.ReadFile(filepath.Join(workDir, s.nodePath(0), "config", "genesis.json"))
+	require.NoError(t, err)
+	return string(content)
+}
+
+// setGenesis copy genesis file to all nodes
+func (s *SystemUnderTest) setGenesis(t *testing.T, srcPath string) {
+	in, err := os.Open(srcPath)
+	require.NoError(t, err)
+	defer in.Close()
+	var buf bytes.Buffer
+
+	_, err = io.Copy(&buf, in)
+	require.NoError(t, err)
+
+	s.withEachNodeHome(func(i int, home string) {
+		require.NoError(t, saveGenesis(home, buf.Bytes()))
+	})
+}
+
+func saveGenesis(home string, content []byte) error {
+	out, err := os.Create(filepath.Join(workDir, home, "config", "genesis.json"))
+	if err != nil {
+		return fmt.Errorf("out file: %w", err)
+	}
+	defer out.Close()
+
+	if _, err = io.Copy(out, bytes.NewReader(content)); err != nil {
+		return fmt.Errorf("write out file: %w", err)
+	}
+
+	if err = out.Close(); err != nil {
+		return fmt.Errorf("close out file: %w", err)
+	}
+	return nil
+}
+
+// ForEachNodeExecAndWait runs the given wasmd commands for all cluster nodes synchronously
+// The commands output is returned for each node.
+func (s *SystemUnderTest) ForEachNodeExecAndWait(t *testing.T, cmds ...[]string) [][]string {
+	result := make([][]string, s.nodesCount)
+	s.withEachNodeHome(func(i int, home string) {
+		result[i] = make([]string, len(cmds))
+		for j, xargs := range cmds {
+			xargs = append(xargs, "--home", home)
+			s.Logf("Execute `wasmd %s`\n", strings.Join(xargs, " "))
+			cmd := exec.Command( //nolint:gosec
+				locateExecutable("wasmd"),
+				xargs...,
+			)
+			cmd.Dir = workDir
+			out, err := cmd.CombinedOutput()
+			require.NoError(t, err, "node %d: %s", i, string(out))
+			s.Logf("Result: %s\n", string(out))
+			result[i][j] = string(out)
+		}
+	})
+	return result
+}
+
+// forEachNodesExecAsync runs the given wasmd command for all cluster nodes and returns without waiting
+func (s *SystemUnderTest) forEachNodesExecAsync(t *testing.T, xargs ...string) []func() error {
+	r := make([]func() error, s.nodesCount)
+	s.withEachNodeHome(func(i int, home string) {
+		args := append(xargs, "--home", home) //nolint:gocritic
+		s.Logf("Execute `wasmd %s`\n", strings.Join(args, " "))
+		cmd := exec.Command( //nolint:gosec
+			locateExecutable("wasmd"),
+			args...,
+		)
+		cmd.Dir = workDir
+		s.watchLogs(i, cmd)
+		require.NoError(t, cmd.Start(), "node %d", i)
+		r[i] = cmd.Wait
+	})
+	return r
+}
+
+func (s SystemUnderTest) withEachNodeHome(cb func(i int, home string)) {
+	for i := 0; i < s.nodesCount; i++ {
+		cb(i, s.nodePath(i))
+	}
+}
+
+// nodePath returns the path of the node within the work dir. not absolute
+func (s SystemUnderTest) nodePath(i int) string {
+	return fmt.Sprintf("%s/node%d/wasmd", s.outputDir, i)
+}
+
+func (s SystemUnderTest) Log(msg string) {
+	if s.verbose {
+		fmt.Fprint(s.out, msg)
+	}
+}
+
+func (s SystemUnderTest) Logf(msg string, args ...interface{}) {
+	s.Log(fmt.Sprintf(msg, args...))
+}
+
+func (s SystemUnderTest) RPCClient(t *testing.T) RPCClient {
+	return NewRPCClient(t, s.rpcAddr)
+}
+
+func (s SystemUnderTest) AllPeers(t *testing.T) []string {
+	result := make([]string, s.nodesCount)
+	for i, n := range s.AllNodes(t) {
+		result[i] = n.PeerAddr()
+	}
+	return result
+}
+
+func (s SystemUnderTest) AllNodes(t *testing.T) []Node {
+	result := make([]Node, s.nodesCount)
+	outs := s.ForEachNodeExecAndWait(t, []string{"tendermint", "show-node-id"})
+	ip, err := server.ExternalIP()
+	require.NoError(t, err)
+
+	for i, out := range outs {
+		result[i] = Node{
+			ID:      strings.TrimSpace(out[0]),
+			IP:      ip,
+			RPCPort: 26657 + i, // as defined in testnet command
+			P2PPort: 16656 + i, // as defined in testnet command
+		}
+	}
+	return result
+}
+
+func (s *SystemUnderTest) resetBuffers() {
+	s.outBuff = ring.New(100)
+	s.errBuff = ring.New(100)
+}
+
+// AddFullnode starts a new fullnode that connects to the existing chain but is not a validator.
+func (s *SystemUnderTest) AddFullnode(t *testing.T, beforeStart ...func(nodeNumber int, nodePath string)) Node {
+	s.MarkDirty()
+	s.nodesCount++
+	nodeNumber := s.nodesCount - 1
+	nodePath := s.nodePath(nodeNumber)
+	_ = os.RemoveAll(nodePath) // drop any legacy path, just in case
+
+	// prepare new node
+	moniker := fmt.Sprintf("node%d", nodeNumber)
+	args := []string{"init", moniker, "--home", nodePath, "--overwrite"}
+	s.Logf("Execute `wasmd %s`\n", strings.Join(args, " "))
+	cmd := exec.Command( //nolint:gosec
+		locateExecutable("wasmd"),
+		args...,
+	)
+	cmd.Dir = workDir
+	s.watchLogs(nodeNumber, cmd)
+	require.NoError(t, cmd.Run(), "failed to start node with id %d", nodeNumber)
+	require.NoError(t, saveGenesis(nodePath, []byte(s.ReadGenesisJSON(t))))
+
+	// quick hack: copy config and overwrite by start params
+	configFile := filepath.Join(workDir, nodePath, "config", "config.toml")
+	_ = os.Remove(configFile)
+	_, err := copyFile(filepath.Join(workDir, s.nodePath(0), "config", "config.toml"), configFile)
+	require.NoError(t, err)
+
+	// start node
+	allNodes := s.AllNodes(t)
+	node := allNodes[len(allNodes)-1]
+	peers := make([]string, len(allNodes)-1)
+	for i, n := range allNodes[0 : len(allNodes)-1] {
+		peers[i] = n.PeerAddr()
+	}
+	for _, c := range beforeStart {
+		c(nodeNumber, nodePath)
+	}
+	args = []string{
+		"start",
+		"--p2p.persistent_peers=" + strings.Join(peers, ","),
+		fmt.Sprintf("--p2p.laddr=tcp://localhost:%d", node.P2PPort),
+		fmt.Sprintf("--rpc.laddr=tcp://localhost:%d", node.RPCPort),
+		fmt.Sprintf("--grpc.address=localhost:%d", 9090+nodeNumber),
+		fmt.Sprintf("--grpc-web.address=localhost:%d", 8090+nodeNumber),
+		"--moniker=" + moniker,
+		"--trace", "--log_level=info",
+		"--home", nodePath,
+	}
+	s.Logf("Execute `wasmd %s`\n", strings.Join(args, " "))
+	cmd = exec.Command( //nolint:gosec
+		locateExecutable("wasmd"),
+		args...,
+	)
+	cmd.Dir = workDir
+	s.watchLogs(nodeNumber, cmd)
+	require.NoError(t, cmd.Start(), "node %d", nodeNumber)
+	return node
+}
+
+// NewEventListener constructor for Eventlistener with system rpc address
+func (s *SystemUnderTest) NewEventListener(t *testing.T) *EventListener {
+	return NewEventListener(t, s.rpcAddr)
+}
+
+type Node struct {
+	ID      string
+	IP      string
+	RPCPort int
+	P2PPort int
+}
+
+func (n Node) PeerAddr() string {
+	return fmt.Sprintf("%s@%s:%d", n.ID, n.IP, n.P2PPort)
+}
+
+func (n Node) RPCAddr() string {
+	return fmt.Sprintf("tcp://%s:%d", n.IP, n.RPCPort)
+}
+
+// locateExecutable looks up the binary on the OS path.
+func locateExecutable(file string) string {
+	path, err := exec.LookPath(file)
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error %s", err.Error()))
+	}
+	if path == "" {
+		panic(fmt.Sprintf("%q not founc", file))
+	}
+	return path
+}
+
+// EventListener watches for events on the chain
+type EventListener struct {
+	t      *testing.T
+	client *client.HTTP
+}
+
+// NewEventListener event listener
+func NewEventListener(t *testing.T, rpcAddr string) *EventListener {
+	httpClient, err := client.New(rpcAddr, "/websocket")
+	require.NoError(t, err)
+	require.NoError(t, httpClient.Start())
+	return &EventListener{client: httpClient, t: t}
+}
+
+var defaultWaitTime = 30 * time.Second
+
+type (
+	CleanupFn     func()
+	EventConsumer func(e ctypes.ResultEvent) (more bool)
+)
+
+// Subscribe to receive events for a topic. Does not block.
+// For query syntax See https://docs.cosmos.network/master/core/events.html#subscribing-to-events
+func (l *EventListener) Subscribe(query string, cb EventConsumer) func() {
+	ctx, done := context.WithCancel(context.Background())
+	l.t.Cleanup(done)
+	eventsChan, err := l.client.WSEvents.Subscribe(ctx, "testing", query)
+	require.NoError(l.t, err)
+	cleanup := func() {
+		ctx, _ := context.WithTimeout(ctx, defaultWaitTime)     //nolint:govet
+		go l.client.WSEvents.Unsubscribe(ctx, "testing", query) //nolint:errcheck
+		done()
+	}
+	go func() {
+		for e := range eventsChan {
+			if !cb(e) {
+				return
+			}
+		}
+	}()
+	return cleanup
+}
+
+// AwaitQuery blocks and waits for a single result or timeout. This can be used with `broadcast-mode=async`.
+// For query syntax See https://docs.cosmos.network/master/core/events.html#subscribing-to-events
+func (l *EventListener) AwaitQuery(query string, optMaxWaitTime ...time.Duration) *ctypes.ResultEvent {
+	c, result := CaptureSingleEventConsumer()
+	maxWaitTime := defaultWaitTime
+	if len(optMaxWaitTime) != 0 {
+		maxWaitTime = optMaxWaitTime[0]
+	}
+	cleanupFn := l.Subscribe(query, TimeoutConsumer(l.t, maxWaitTime, c))
+	l.t.Cleanup(cleanupFn)
+	return result
+}
+
+// TimeoutConsumer is an event consumer decorator with a max wait time. Panics when wait time exceeded without
+// a result returned
+func TimeoutConsumer(t *testing.T, maxWaitTime time.Duration, next EventConsumer) EventConsumer {
+	ctx, done := context.WithCancel(context.Background())
+	t.Cleanup(done)
+	timeout := time.NewTimer(maxWaitTime)
+	timedOut := make(chan struct{}, 1)
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-timeout.C:
+			timedOut <- struct{}{}
+			close(timedOut)
+		}
+	}()
+	return func(e ctypes.ResultEvent) (more bool) {
+		select {
+		case <-timedOut:
+			t.Fatalf("Timeout waiting for new events %s", maxWaitTime)
+			return false
+		default:
+			timeout.Reset(maxWaitTime)
+			result := next(e)
+			if !result {
+				done()
+			}
+			return result
+		}
+	}
+}
+
+// CaptureSingleEventConsumer consumes one event. No timeout
+func CaptureSingleEventConsumer() (EventConsumer, *ctypes.ResultEvent) {
+	var result ctypes.ResultEvent
+	return func(e ctypes.ResultEvent) (more bool) {
+		return false
+	}, &result
+}
+
+// CaptureAllEventsConsumer is an `EventConsumer` that captures all events until `done()` is called to stop or timeout happens.
+// The consumer works async in the background and returns all the captured events when `done()` is called.
+// This can be used to verify that certain events have happened.
+// Example usage:
+//
+//		c, done := CaptureAllEventsConsumer(t)
+//		query := `tm.event='Tx'`
+//		cleanupFn := l.Subscribe(query, c)
+//		t.Cleanup(cleanupFn)
+//
+//	 // do something in your test that create events
+//
+//		assert.Len(t, done(), 1) // then verify your assumption
+func CaptureAllEventsConsumer(t *testing.T, optMaxWaitTime ...time.Duration) (c EventConsumer, done func() []ctypes.ResultEvent) {
+	maxWaitTime := defaultWaitTime
+	if len(optMaxWaitTime) != 0 {
+		maxWaitTime = optMaxWaitTime[0]
+	}
+	var (
+		mu             sync.Mutex
+		capturedEvents []ctypes.ResultEvent
+		exit           bool
+	)
+	collectEventsConsumer := func(e ctypes.ResultEvent) (more bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		if exit {
+			return false
+		}
+		capturedEvents = append(capturedEvents, e)
+		return true
+	}
+
+	return TimeoutConsumer(t, maxWaitTime, collectEventsConsumer), func() []ctypes.ResultEvent {
+		mu.Lock()
+		defer mu.Unlock()
+		exit = true
+		return capturedEvents
+	}
+}
+
+// restoreOriginalGenesis replace nodes genesis by the one created on setup
+func restoreOriginalGenesis(t *testing.T, s SystemUnderTest) {
+	src := filepath.Join(workDir, s.nodePath(0), "config", "genesis.json.orig")
+	s.setGenesis(t, src)
+}
+
+// restoreOriginalKeyring replaces test keyring with original
+func restoreOriginalKeyring(t *testing.T, s SystemUnderTest) {
+	dest := filepath.Join(workDir, s.outputDir, "keyring-test")
+	require.NoError(t, os.RemoveAll(dest))
+	for i := 0; i < s.initialNodesCount; i++ {
+		src := filepath.Join(workDir, s.nodePath(i), "keyring-test")
+		require.NoError(t, copyFilesInDir(src, dest))
+	}
+}
+
+// copyFile copy source file to dest file path
+func copyFile(src, dest string) (*os.File, error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return nil, err
+	}
+	defer in.Close()
+	out, err := os.Create(dest)
+	if err != nil {
+		return nil, err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	return out, err
+}
+
+// copyFilesInDir copy files in src dir to dest path
+func copyFilesInDir(src, dest string) error {
+	err := os.MkdirAll(dest, 0o755)
+	if err != nil {
+		return fmt.Errorf("mkdirs: %s", err)
+	}
+	fs, err := ioutil.ReadDir(src)
+	if err != nil {
+		return fmt.Errorf("read dir: %s", err)
+	}
+	for _, f := range fs {
+		if f.IsDir() {
+			continue
+		}
+		if _, err := copyFile(filepath.Join(src, f.Name()), filepath.Join(dest, f.Name())); err != nil {
+			return fmt.Errorf("copy file: %q: %s", f.Name(), err)
+		}
+	}
+	return nil
+}
+
+func storeTempFile(t *testing.T, content []byte) *os.File {
+	out, err := ioutil.TempFile(t.TempDir(), "genesis")
+	require.NoError(t, err)
+	_, err = io.Copy(out, bytes.NewReader(content))
+	require.NoError(t, err)
+	require.NoError(t, out.Close())
+	return out
+}

tests/system/testdata/download_releases.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -o errexit -o nounset -o pipefail
+command -v shellcheck > /dev/null && shellcheck "$0"
+
+if [ $# -ne 1 ]; then
+  echo "Usage: ./download_releases.sh RELEASE_TAG"
+  exit 1
+fi
+
+tag="$1"
+
+for contract in hackatom reflect; do
+  url="https://github.com/CosmWasm/cosmwasm/releases/download/$tag/${contract}.wasm"
+  echo "Downloading $url ..."
+  wget -O "${contract}.wasm" "$url"
+
+  # create the zip variant
+  gzip -k "${contract}.wasm"
+  mv "${contract}.wasm.gz" "${contract}.wasm.gzip"
+  rm -f "${contract}.wasm"
+done
+
+rm -f version.txt
+echo "$tag" >version.txt

tests/system/testdata/version.txt

@@ -0,0 +1 @@
+v1.2.3

x/wasm/client/cli/utils.go

@@ -0,0 +1,34 @@
+package cli
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/cosmos/cosmos-sdk/server"
+	"github.com/spf13/cobra"
+
+	tmcmd "github.com/cometbft/cometbft/cmd/cometbft/commands"
+)
+
+// ExtendUnsafeResetAllCmd - also clear wasm dir
+func ExtendUnsafeResetAllCmd(rootCmd *cobra.Command) {
+	unsafeResetCmd := tmcmd.ResetAllCmd.Use
+	for _, branchCmd := range rootCmd.Commands() {
+		if branchCmd.Use != "tendermint" {
+			continue
+		}
+		for _, cmd := range branchCmd.Commands() {
+			if cmd.Use == unsafeResetCmd {
+				serverRunE := cmd.RunE
+				cmd.RunE = func(cmd *cobra.Command, args []string) error {
+					if err := serverRunE(cmd, args); err != nil {
+						return nil
+					}
+					serverCtx := server.GetServerContextFromCmd(cmd)
+					return os.RemoveAll(filepath.Join(serverCtx.Config.RootDir, "wasm"))
+				}
+				return
+			}
+		}
+	}
+}