Separate anvil and ganache tests (#307)

.github/workflows/ci.yml

@@ -202,8 +202,8 @@ jobs:
           report-name: "wallet"
 
 
-  contractkit-tests:
-    name: ContractKit Tests
+  contractkit-tests-anvil:
+    name: ContractKit Tests (Anvil)
     runs-on: ['self-hosted', 'org', '8-cpu']
     timeout-minutes: 30
     permissions: # Must change the job token permissions to use Akeyless JWT auth
@@ -223,13 +223,39 @@ jobs:
           version: "nightly-f625d0fa7c51e65b4bf1e8f7931cd1c6e2e285e9"
       - name: Run tests
         run: |
-          yarn workspace @celo/contractkit test --coverage
+          yarn workspace @celo/contractkit test-anvil --coverage
       - uses: ./.github/actions/upload-codecov
         with:
-          report-name: "contractkit"
+          report-name: "contractkit-anvil"
 
-  cli-tests:
-    name: CeloCli Tests
+  contractkit-tests-ganache:
+    name: ContractKit Tests (Ganache)
+    runs-on: ['self-hosted', 'org', '8-cpu']
+    timeout-minutes: 30
+    permissions: # Must change the job token permissions to use Akeyless JWT auth
+      id-token: write
+      contents: read
+    needs: [install-dependencies]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Sync workspace
+        uses: ./.github/actions/sync-workspace
+        with:
+          artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }}
+      - run: sudo corepack enable yarn
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: "nightly-f625d0fa7c51e65b4bf1e8f7931cd1c6e2e285e9"
+      - name: Run tests
+        run: |
+          yarn workspace @celo/contractkit test-ganache --coverage
+      - uses: ./.github/actions/upload-codecov
+        with:
+          report-name: "contractkit-ganache"
+
+  cli-tests-anvil:
+    name: CeloCli Tests (Anvil)
     runs-on: ['self-hosted', 'org', '8-cpu']
     timeout-minutes: 30
     permissions: # Must change the job token permissions to use Akeyless JWT auth
@@ -248,7 +274,7 @@ jobs:
           version: ${{ env.SUPPORTED_FOUNDRY_VERSION }}
       - name: Run tests
         run: |
-          yarn workspace @celo/celocli test-ci --coverage
+          yarn workspace @celo/celocli test-ci-anvil --coverage
       - name: Verify that a new account can be created
         run: |
           yarn workspace @celo/celocli run celocli account:new
@@ -257,8 +283,33 @@ jobs:
           yarn workspace @celo/celocli run celocli releasecelo --help
       - uses: ./.github/actions/upload-codecov
         with:
-          report-name: "celocli"
+          report-name: "celocli-anvil"
 
+  cli-tests-ganache:
+    name: CeloCli Tests (Ganache)
+    runs-on: ['self-hosted', 'org', '8-cpu']
+    timeout-minutes: 30
+    permissions: # Must change the job token permissions to use Akeyless JWT auth
+      id-token: write
+      contents: read
+    needs: [install-dependencies]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Sync workspace
+        uses: ./.github/actions/sync-workspace
+        with:
+          artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }}
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: ${{ env.SUPPORTED_FOUNDRY_VERSION }}
+      - name: Run tests
+        run: |
+          yarn workspace @celo/celocli test-ci-ganache --coverage
+      - uses: ./.github/actions/upload-codecov
+        with:
+          report-name: "celocli-ganache"
+
   docs-tests:
     name: Docs tests
     runs-on: ['self-hosted', 'org', '8-cpu']

docs/sdk/contractkit/modules/test_utils_setup_global.md

@@ -20,4 +20,4 @@
 
 #### Defined in
 
-[packages/sdk/contractkit/src/test-utils/setup.global.ts:9](https://github.com/celo-org/developer-tooling/blob/master/packages/sdk/contractkit/src/test-utils/setup.global.ts#L9)
+[packages/sdk/contractkit/src/test-utils/setup.global.ts:10](https://github.com/celo-org/developer-tooling/blob/master/packages/sdk/contractkit/src/test-utils/setup.global.ts#L10)

docs/sdk/contractkit/modules/test_utils_teardown_global.md

@@ -20,4 +20,4 @@
 
 #### Defined in
 
-[packages/sdk/contractkit/src/test-utils/teardown.global.ts:3](https://github.com/celo-org/developer-tooling/blob/master/packages/sdk/contractkit/src/test-utils/teardown.global.ts#L3)
+[packages/sdk/contractkit/src/test-utils/teardown.global.ts:4](https://github.com/celo-org/developer-tooling/blob/master/packages/sdk/contractkit/src/test-utils/teardown.global.ts#L4)

packages/cli/DEVELOP.md

@@ -0,0 +1,22 @@
+# Developer Guide
+
+## Running Tests
+
+### All tests
+
+`yarn test`
+
+### Anvil & Ganache tests
+
+There are 2 more yarn commands:
+
+- `yarn test-anvil`
+- `yarn test-ganache`
+
+They will run anvil and ganache tests (+ common tests that are neither of those) ONLY by providing necessary process env variables. It determines which tests to run based on `RUN_ANVIL_TESTS` env var which which is used by `testWithAnvil` (from [packages/dev-utils/src/anvil-test.ts](packages/dev-utils/src/anvil-test.ts)) and `RUN_GANACHE_TESTS` env var wich is used `testWithGanache` (from [packages/dev-utils/src/ganache-test.ts](packages/dev-utils/src/ganache-test.ts)) to call `testWithWeb3` (from [packages/dev-utils/src/test-utils.ts](packages/dev-utils/src/test-utils.ts)) with required parameters. See docs for `testWithWeb3` for a detailed description of the logic on how it runs only the desired tests.
+
+Please note that anvil tests will be run in parallel and ganache tests will be run sequentially (by providing `--runInBand` flag).
+
+#### CI
+
+There are 2 CI equivalents of the `test-*` commands that supply ` --workerIdleMemoryLimit=0.1` flags that are required for them to work in CI environment.

packages/cli/package.json

@@ -30,7 +30,11 @@
     "prepublish": "",
     "prepack": "yarn run build && oclif manifest && oclif readme",
     "test": "TZ=UTC NODE_OPTIONS='--experimental-vm-modules' yarn jest --runInBand --forceExit",
-    "test-ci": "TZ=UTC NODE_OPTIONS='--experimental-vm-modules' yarn jest --runInBand --workerIdleMemoryLimit=0.1 --forceExit"
+    "test-anvil": "RUN_GANACHE_TESTS=false RUN_ANVIL_TESTS=true TZ=UTC NODE_OPTIONS='--experimental-vm-modules' yarn jest --forceExit",
+    "test-ganache": "RUN_GANACHE_TESTS=true RUN_ANVIL_TESTS=false TZ=UTC NODE_OPTIONS='--experimental-vm-modules' yarn jest --runInBand --forceExit",
+    "test-ci": "TZ=UTC NODE_OPTIONS='--experimental-vm-modules' yarn jest --runInBand --workerIdleMemoryLimit=0.1 --forceExit",
+    "test-ci-anvil": "yarn test-anvil --workerIdleMemoryLimit=0.1",
+    "test-ci-ganache": "yarn test-ganache --workerIdleMemoryLimit=0.1"
   },
   "dependencies": {
     "@celo/abis": "11.0.0",

packages/cli/src/commands/election/list.test.ts

@@ -1,12 +1,14 @@
 import { ElectionWrapper, ValidatorGroupVote } from '@celo/contractkit/lib/wrappers/Election'
+import { testWithGanache } from '@celo/dev-utils/lib/ganache-test'
 import { ux } from '@oclif/core'
 import BigNumber from 'bignumber.js'
-import { testLocally } from '../../test-utils/cliUtils'
+import Web3 from 'web3'
+import { testLocallyWithWeb3Node } from '../../test-utils/cliUtils'
 import ElectionList from './list'
 
 process.env.NO_SYNCCHECK = 'true'
 
-describe('election:list cmd', () => {
+testWithGanache('election:list cmd', (web3: Web3) => {
   test('shows list when no arguments provided', async () => {
     const getValidatorGroupsVotesMock = jest.spyOn(
       ElectionWrapper.prototype,
@@ -33,7 +35,7 @@ describe('election:list cmd', () => {
 
     const writeMock = jest.spyOn(ux.write, 'stdout')
 
-    await testLocally(ElectionList, ['--csv'])
+    await testLocallyWithWeb3Node(ElectionList, ['--csv'], web3)
 
     expect(getValidatorGroupsVotesMock).toHaveBeenCalled()
     expect(writeMock.mock.calls).toMatchInlineSnapshot(`

packages/cli/src/commands/identity/get-attestations.test.ts

@@ -1,34 +1,39 @@
-import { testLocally } from '../../test-utils/cliUtils'
+import { testWithGanache } from '@celo/dev-utils/lib/ganache-test'
+import Web3 from 'web3'
+import { testLocallyWithWeb3Node } from '../../test-utils/cliUtils'
 import GetAttestations from './get-attestations'
 
 process.env.NO_SYNCCHECK = 'true'
 
-describe('identity:get-attetstations', () => {
+testWithGanache('identity:get-attetstations', (web3: Web3) => {
   describe('input validation correctly outputs errors', () => {
     const consoleOutput: string[] = []
     const mockedError = (output: string) => consoleOutput.push(output)
     beforeEach(() => (console.error = mockedError))
 
     it('Fails when neither from, pepper, nor identifier are specified', async () => {
-      await expect(testLocally(GetAttestations, ['--phoneNumber', '+15555555555'])).rejects.toThrow(
-        'Must specify either --from or --pepper or --identifier'
-      )
+      await expect(
+        testLocallyWithWeb3Node(GetAttestations, ['--phoneNumber', '+15555555555'], web3)
+      ).rejects.toThrow('Must specify either --from or --pepper or --identifier')
     })
 
     it('Fails when neither phone number nor identifier are specified', async () => {
       await expect(
-        testLocally(GetAttestations, ['--from', '0x47e172F6CfB6c7D01C1574fa3E2Be7CC73269D95'])
+        testLocallyWithWeb3Node(
+          GetAttestations,
+          ['--from', '0x47e172F6CfB6c7D01C1574fa3E2Be7CC73269D95'],
+          web3
+        )
       ).rejects.toThrow('Must specify phoneNumber if identifier not provided')
     })
 
     it('Successfully prints identifier when given pepper and number', async () => {
       console.log = jest.fn()
-      await testLocally(GetAttestations, [
-        '--phoneNumber',
-        '+15555555555',
-        '--pepper',
-        'XQke2bjvN7mPt',
-      ])
+      await testLocallyWithWeb3Node(
+        GetAttestations,
+        ['--phoneNumber', '+15555555555', '--pepper', 'XQke2bjvN7mPt'],
+        web3
+      )
       expect(console.log).toHaveBeenCalledWith(
         'Identifier: 0xd9460ae529b2889716c8f1ccebb5efec945adc46fe1e9cd16f6242463e81f37c'
       )

packages/cli/src/test-utils/setup.global.ts

@@ -1,4 +1,5 @@
 import baseSetup from '@celo/dev-utils/lib/ganache-setup'
+import { shouldRunGanacheTests } from '@celo/dev-utils/lib/ganache-test'
 // Has to import the matchers somewhere so that typescript knows the matchers have been made available
 import _unused from '@celo/dev-utils/lib/matchers'
 import * as path from 'path'
@@ -7,13 +8,19 @@ import * as path from 'path'
 // If there is not, then your editor probably deleted it automatically.
 
 export default async function globalSetup() {
-  console.log('\nstarting ganache...')
-  const chainDataPath = path.join(path.dirname(require.resolve('@celo/celo-devchain')), '../chains')
-  // v X refers to core contract release X
-  await baseSetup(path.resolve(chainDataPath), 'v11.tar.gz', {
-    from_targz: true,
-  })
-  console.log('\n ganache started...')
+  if (shouldRunGanacheTests()) {
+    console.log('\nstarting ganache...')
+    const chainDataPath = path.join(
+      path.dirname(require.resolve('@celo/celo-devchain')),
+      '../chains'
+    )
+    // v X refers to core contract release X
+    await baseSetup(path.resolve(chainDataPath), 'v11.tar.gz', {
+      from_targz: true,
+    })
+    console.log('\n ganache started...')
+  }
+
   // it is necessary to disabled oclif integration with ts-node as
   // together it leads to a silent signit error and exit when tsconfk is loaded.
   // @ts-ignore - because global this doesnt have oclif property

packages/cli/src/test-utils/teardown.global.ts

@@ -1,5 +1,8 @@
 import teardown from '@celo/dev-utils/lib/ganache-teardown'
+import { shouldRunGanacheTests } from '@celo/dev-utils/lib/ganache-test'
 
 export default async function globalTeardown() {
-  await teardown()
+  if (shouldRunGanacheTests()) {
+    await teardown()
+  }
 }

packages/dev-utils/src/anvil-test.ts

@@ -44,11 +44,15 @@ export function testWithAnvil(name: string, fn: (web3: Web3) => void) {
 
   // for each test suite, we start and stop a new anvil instance
   return testWithWeb3(name, `http://127.0.0.1:${anvil.port}`, fn, {
-    beforeAll: async () => {
-      await anvil.start()
-    },
-    afterAll: async () => {
-      await anvil.stop()
+    runIf:
+      process.env.RUN_ANVIL_TESTS === 'true' || typeof process.env.RUN_ANVIL_TESTS === 'undefined',
+    hooks: {
+      beforeAll: async () => {
+        await anvil.start()
+      },
+      afterAll: async () => {
+        await anvil.stop()
+      },
     },
   })
 }

packages/dev-utils/src/ganache-test.ts

@@ -16,7 +16,15 @@ export async function mineBlocks(blocks: number, web3: Web3) {
 }
 
 export function testWithGanache(name: string, fn: (web3: Web3) => void) {
-  return testWithWeb3(name, 'http://localhost:8545', fn)
+  return testWithWeb3(name, 'http://localhost:8545', fn, {
+    runIf: shouldRunGanacheTests(),
+  })
+}
+
+export function shouldRunGanacheTests(): boolean {
+  return (
+    process.env.RUN_GANACHE_TESTS === 'true' || typeof process.env.RUN_GANACHE_TESTS === 'undefined'
+  )
 }
 
 /**

packages/dev-utils/src/test-utils.ts

@@ -58,11 +58,23 @@ type TestWithWeb3Hooks = {
   afterAll?: () => Promise<void>
 }
 
+/**
+ * Creates a test suite with a given name and provides function with a web3 instance connected to the given rpcUrl.
+ *
+ * It is an equivalent of jest `describe` with the web3 additioon. It also provides hooks for beforeAll and afterAll.
+ *
+ * Optionally if a runIf flag is set to false the test suite will be skipped (useful for conditional test suites). By
+ * default all test suites are run normally, but if the runIf flag is set to false the test suite will be skipped by using
+ * jest `describe.skip`. It will be reported in the summary as "skipped".
+ */
 export function testWithWeb3(
   name: string,
   rpcUrl: string,
   fn: (web3: Web3) => void,
-  hooks?: TestWithWeb3Hooks
+  options: {
+    hooks?: TestWithWeb3Hooks
+    runIf?: boolean
+  } = {}
 ) {
   const web3 = new Web3(rpcUrl)
 
@@ -71,11 +83,19 @@ export function testWithWeb3(
   // before polling again making the tests slow
   web3.eth.transactionPollingInterval = 10
 
-  describe(name, () => {
+  // By default we run all the tests
+  let describeFn = describe
+
+  // and only skip them if explicitly stated
+  if (options.runIf === false) {
+    describeFn = describe.skip
+  }
+
+  describeFn(name, () => {
     let snapId: string | null = null
 
-    if (hooks?.beforeAll) {
-      beforeAll(hooks.beforeAll)
+    if (options.hooks?.beforeAll) {
+      beforeAll(options.hooks.beforeAll)
     }
 
     beforeEach(async () => {
@@ -89,9 +109,9 @@ export function testWithWeb3(
       if (snapId != null) {
         await evmRevert(web3, snapId)
       }
-      if (hooks?.afterAll) {
+      if (options.hooks?.afterAll) {
         // hook must be awaited here or jest doesnt actually wait for it and complains of open handles
-        await hooks.afterAll()
+        await options.hooks.afterAll()
       }
     })
 

packages/sdk/contractkit/DEVELOP.md

@@ -2,8 +2,21 @@
 
 ## Running Tests
 
+### All tests
+
 `yarn test`
 
+### Anvil & Ganache tests
+
+There are 2 more yarn commands:
+
+- `yarn test-anvil`
+- `yarn test-ganache`
+
+They will run anvil and ganache tests (+ common tests that are neither of those) ONLY by providing necessary process env variables. It determines which tests to run based on `RUN_ANVIL_TESTS` env var which which is used by `testWithAnvil` (from [packages/dev-utils/src/anvil-test.ts](packages/dev-utils/src/anvil-test.ts)) and `RUN_GANACHE_TESTS` env var wich is used `testWithGanache` (from [packages/dev-utils/src/ganache-test.ts](packages/dev-utils/src/ganache-test.ts)) to call `testWithWeb3` (from [packages/dev-utils/src/test-utils.ts](packages/dev-utils/src/test-utils.ts)) with required parameters. See docs for `testWithWeb3` for a detailed description of the logic on how it runs only the desired tests.
+
+Please note that anvil tests will be run in parallel and ganache tests will be run sequentially (by providing `--runInBand` flag).
+
 ### Testing Azure HSM Signer
 
 The tests include an in-memory mock implementation which tests a majority of the functionality. In the case that changes are made to any of the wallet/signing logic, the HSM signer should be tested end-to-end by using an Azure KeyVault. To test against the KeyVault, environment variables are used to provide the client ID and client secret for authentication. Please see the .env file for the required variables. After deploying your KeyVault and generating an ECDSA-SECP256k1 key, you must create a service principal account and provide it signing access to the KeyVault. Instructions on how to do this can be found here: <https://www.npmjs.com/package/@azure/keyvault-keys#configuring-your-key-vault>. If the .env variables are specified, the tests will automatically switch from using the mock client to the actual KeyVault.

packages/sdk/contractkit/package.json

@@ -21,6 +21,8 @@
     "prepublishOnly": "yarn build",
     "docs": "yarn run --top-level typedoc",
     "test": "NODE_OPTIONS=--experimental-vm-modules yarn run --top-level jest --runInBand --forceExit",
+    "test-anvil": "RUN_GANACHE_TESTS=false RUN_ANVIL_TESTS=true NODE_OPTIONS=--experimental-vm-modules yarn run --top-level jest --forceExit",
+    "test-ganache": "RUN_GANACHE_TESTS=true RUN_ANVIL_TESTS=false NODE_OPTIONS=--experimental-vm-modules yarn run --top-level jest --runInBand --forceExit",
     "lint": "yarn run --top-level eslint -c .eslintrc.js "
   },
   "dependencies": {