fix: simulation tests

.github/workflows/sims.yml

@@ -31,6 +31,24 @@ jobs:
           path: ~/go/bin
           key: ${{ runner.os }}-go-runsim-binary
 
+  test-sim-nondeterminism:
+    runs-on: ubuntu-latest
+    needs: [build]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+          check-latest: true
+      - uses: actions/cache@v3
+        with:
+          path: ~/go/bin
+          key: ${{ runner.os }}-go-runsim-binary
+      - name: test-sim-import-export
+        run: |
+          make test-sim-nondeterminism
+
   test-sim-import-export:
     runs-on: ubuntu-latest
     needs: [build]

Makefile

@@ -3,7 +3,8 @@
 DOCKER := $(shell which docker)
 
 export GO111MODULE = on
-SIMAPP = ./simapp
+SIMAPP = ./simapp/app/
+BINDIR ?= $(GOPATH)/bin
 
 ###############################################################################
 ###                                     e2e                                 ###
@@ -76,6 +77,7 @@ lint-fix:
 GO := go
 TARGET := cada
 BINDIR ?= $(GOPATH)/bin
+CURRENT_DIR = $(shell pwd)
 
 .PHONY: all build install clean
 
@@ -142,9 +144,12 @@ endif
 
 .PHONY: run-tests test test-all $(TEST_TARGETS)
 
+runsim:
+	go install github.com/cosmos/tools/cmd/[email protected]
+
 test-sim-nondeterminism:
 	@echo "Running non-determinism test..."
-	@go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true \
+	@cd ${CURRENT_DIR}/simapp/app && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
 		-NumBlocks=20 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h
 
 test-sim-custom-genesis-fast:
@@ -153,13 +158,13 @@ test-sim-custom-genesis-fast:
 	@go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation -Genesis=${HOME}/.cada/config/genesis.json \
 		-Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h
 
-test-sim-import-export: runsim
-	@echo "Running application import/export simulation. This may take several minutes..."
-	@$(BINDIR)/runsim -Jobs=4 -SimAppPkg=$(APP) -ExitOnFail 50 5 TestAppImportExport
+# test-sim-import-export: runsim
+# 	@echo "Running application import/export simulation. This may take several minutes..."
+# 	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppImportExport
 
 test-sim-after-import: runsim
 	@echo "Running application simulation-after-import. This may take several minutes..."
-	@$(BINDIR)/runsim -Jobs=4 -SimAppPkg=$(APP) -ExitOnFail 50 5 TestAppSimulationAfterImport
+	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppSimulationAfterImport
 
 test-sim-custom-genesis-multi-seed: runsim
 	@echo "Running multi-seed custom genesis simulation..."

simapp/app/app.go

@@ -893,7 +893,7 @@ func NewChainApp(
 	// NOTE: this is not required apps that don't use the simulator for fuzz testing
 	// transactions
 	overrideModules := map[string]module.AppModuleSimulation{
-		authtypes.ModuleName: auth.NewAppModule(app.appCodec, app.AccountKeeper, authsims.RandomGenesisAccounts, app.GetSubspace(authtypes.ModuleName)),
+		authtypes.ModuleName: auth.NewAppModule(app.appCodec, app.AccountKeeper, authsims.RandomGenesisAccounts, nil),
 	}
 	app.sm = module.NewSimulationManagerFromAppModules(app.ModuleManager.Modules, overrideModules)
 

simapp/app/sim_bench_test.go

@@ -0,0 +1,164 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
+	flag "github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cosmos/cosmos-sdk/baseapp"
+	"github.com/cosmos/cosmos-sdk/client/flags"
+	"github.com/cosmos/cosmos-sdk/server"
+	simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
+	simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
+	"github.com/cosmos/cosmos-sdk/x/simulation"
+	simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
+)
+
+var FlagEnableBenchStreamingValue bool
+
+// Get flags every time the simulator is run
+func init() {
+	flag.BoolVar(&FlagEnableBenchStreamingValue, "EnableStreaming", false, "Enable streaming service")
+}
+
+// Profile with:
+// /usr/local/go/bin/go test -benchmem -run=^$ cosmossdk.io/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
+func BenchmarkFullAppSimulation(b *testing.B) {
+	b.ReportAllocs()
+
+	config := simcli.NewConfigFromFlags()
+	config.ChainID = SimAppChainID
+
+	db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
+	if err != nil {
+		b.Fatalf("simulation setup failed: %s", err.Error())
+	}
+
+	if skip {
+		b.Skip("skipping benchmark application simulation")
+	}
+
+	defer func() {
+		require.NoError(b, db.Close())
+		require.NoError(b, os.RemoveAll(dir))
+	}()
+
+	appOptions := viper.New()
+	if FlagEnableStreamingValue {
+		m := make(map[string]interface{})
+		m["streaming.abci.keys"] = []string{"*"}
+		m["streaming.abci.plugin"] = "abci_v1"
+		m["streaming.abci.stop-node-on-err"] = true
+		for key, value := range m {
+			appOptions.SetDefault(key, value)
+		}
+	}
+	appOptions.SetDefault(flags.FlagHome, DefaultNodeHome)
+	appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue)
+
+	app := NewChainApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID))
+
+	// run randomized simulation
+	_, simParams, simErr := simulation.SimulateFromSeed(
+		b,
+		os.Stdout,
+		app.BaseApp,
+		simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
+		simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
+		simtestutil.SimulationOperations(app, app.AppCodec(), config),
+		BlockedAddresses(),
+		config,
+		app.AppCodec(),
+	)
+
+	// export state and simParams before the simulation error is checked
+	if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil {
+		b.Fatal(err)
+	}
+
+	if simErr != nil {
+		b.Fatal(simErr)
+	}
+
+	if config.Commit {
+		simtestutil.PrintStats(db)
+	}
+}
+
+func BenchmarkInvariants(b *testing.B) {
+	b.ReportAllocs()
+
+	config := simcli.NewConfigFromFlags()
+	config.ChainID = SimAppChainID
+
+	db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-invariant-bench", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
+	if err != nil {
+		b.Fatalf("simulation setup failed: %s", err.Error())
+	}
+
+	if skip {
+		b.Skip("skipping benchmark application simulation")
+	}
+
+	config.AllInvariants = false
+
+	defer func() {
+		require.NoError(b, db.Close())
+		require.NoError(b, os.RemoveAll(dir))
+	}()
+
+	appOptions := make(simtestutil.AppOptionsMap, 0)
+	appOptions[flags.FlagHome] = DefaultNodeHome
+	appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue
+
+	app := NewChainApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID))
+
+	// run randomized simulation
+	_, simParams, simErr := simulation.SimulateFromSeed(
+		b,
+		os.Stdout,
+		app.BaseApp,
+		simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
+		simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
+		simtestutil.SimulationOperations(app, app.AppCodec(), config),
+		BlockedAddresses(),
+		config,
+		app.AppCodec(),
+	)
+
+	// export state and simParams before the simulation error is checked
+	if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil {
+		b.Fatal(err)
+	}
+
+	if simErr != nil {
+		b.Fatal(simErr)
+	}
+
+	if config.Commit {
+		simtestutil.PrintStats(db)
+	}
+
+	ctx := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight() + 1})
+
+	// 3. Benchmark each invariant separately
+	//
+	// NOTE: We use the crisis keeper as it has all the invariants registered with
+	// their respective metadata which makes it useful for testing/benchmarking.
+	for _, cr := range app.CrisisKeeper.Routes() {
+		cr := cr
+		b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
+			if res, stop := cr.Invar(ctx); stop {
+				b.Fatalf(
+					"broken invariant at block %d of %d\n%s",
+					ctx.BlockHeight()-1, config.NumBlocks, res,
+				)
+			}
+		})
+	}
+}