Merge branch 'develop' into patch-1

.github/workflows/branch-release.yml

@@ -46,6 +46,7 @@ jobs:
       GIT_BRANCH: ${{ steps.set_vars.outputs.GIT_BRANCH }}
       RELEASE_BRANCH_PATTERN: ${{ steps.set_vars.outputs.RELEASE_BRANCH_PATTERN }}
       RELEASE_TAG: ${{ steps.set_vars.outputs.RELEASE_TAG }}
+      GIT_BRANCH_UNFORMATTED: ${{ steps.branch-name.outputs.current_branch }}
     steps:
       - name: Get branch name
         id: branch-name
@@ -114,9 +115,19 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           env: pr-${{ github.event.number }}
 
+  generate-types:
+    name: Generate types
+    needs: [ init, build-and-test ]
+    uses: ./.github/workflows/reusable-generate-types.yml
+    secrets: inherit
+    with:
+      branch: ${{ needs.init.outputs.GIT_BRANCH_UNFORMATTED }}
+      parachainDocker: ${{ github.event.inputs.parachainDocker }}
+      globalVersion: ${{ needs.init.outputs.GLOBAL_VERSION }}
+
   run-e2e-test:
     name: Run e2e tests
-    needs: [init, validate-branch-name, build-and-test]
+    needs: [init, validate-branch-name, build-and-test, generate-types]
     uses: ./.github/workflows/reusable-e2e-tests.yml
     secrets: inherit
     permissions:

.github/workflows/reusable-build-and-test.yml

@@ -30,13 +30,13 @@ permissions:
 jobs:
   build-node-image:
     name: Build Docker image
-    runs-on: [compile]
+    runs-on: [compile-gke]
     container:
       image: ${{ inputs.builder_image }}
-      # env:
-      #   DOCKER_HOST: "unix:///run/docker/docker.sock"
-      # volumes:
-      #   - /run/docker:/run/docker
+      env:
+        DOCKER_HOST: "unix:///run/docker/docker.sock"
+      volumes:
+        - /run/docker:/run/docker
     steps:
       - uses: actions/checkout@v3
       - name: Authenticate to Google Cloud
@@ -150,7 +150,7 @@ jobs:
 
   unit-test:
     name: Unit tests
-    runs-on: [compile]
+    runs-on: [compile-gke]
     container:
       image: ${{ inputs.builder_image }}
     steps:
@@ -178,7 +178,7 @@ jobs:
 
   coverage-report:
     name: Coverage report
-    runs-on: [compile]
+    runs-on: [compile-gke]
     container:
       image: ${{ inputs.builder_image }}
       options: --security-opt seccomp=unconfined
@@ -243,7 +243,7 @@ jobs:
 
   build-and-run-try-runtime:
     name: Run try-runtime checks
-    runs-on: [compile]
+    runs-on: [compile-gke]
     container:
       image: ${{ inputs.builder_image }}
     steps:
@@ -280,7 +280,7 @@ jobs:
   run-benchmarks:
     name: Run runtime benchmarks
     # `performance` self-hosted runners have 8 cores and 16GB of RAM
-    runs-on: [compile]
+    runs-on: [performance-gke]
     env:
       STEPS: 2
       REPEATS: 1
@@ -346,7 +346,7 @@ jobs:
 
   build-wasms:
     name: Export wasm artifacts
-    runs-on: [compile]
+    runs-on: [compile-gke]
     env:
       STEPS: 2
       REPEATS: 1

.github/workflows/reusable-e2e-tests.yml

@@ -126,7 +126,7 @@ jobs:
           - command: "yarn test-parallel-3rdPartyRewards"
             fast: true
 
-    runs-on: [compile]
+    runs-on: [e2e-gke]
     timeout-minutes: 180
     env:
       API_URL: "ws://127.0.0.1:9946"
@@ -288,10 +288,6 @@ jobs:
         run: |
           docker kill $(docker ps -q) 2>/dev/null && echo $?
           docker-compose down -v
-      
-      - name: Fix permissions on self-hosted runner
-        if: always()
-        run: chown -R 1100:1100 $GITHUB_WORKSPACE
 
   test-complete:
     needs: [setup-report, e2e-test-matrix]

.github/workflows/reusable-perfomance-tests.yml

@@ -22,7 +22,7 @@ jobs:
   performance-tests:
     # Allows to keep e2e tests jobs running even if performance-tests fail
     continue-on-error: true
-    runs-on: [compile]
+    runs-on: [compile-gke]
     timeout-minutes: 180
     env:
       ENV_REF: ${{ inputs.targetEnv || format('pr-{0}', github.event.number) }}

Mangata.md

@@ -0,0 +1,66 @@
+# Block construction/execution
+Mangata implements front-running bots prevention on automated market maker crypto exchange. For that reason it was decided to split block production and block execution into two following blocks. Execution of the transaction included in block N is delayed by 1 block comparing to origin substrate implementation. It is following block producer (N+1) who calculates the order of transactions execution from the previous block. This way none is able to foresee in what order transactions will be performed. 
+
+Affected creates:
+- [sc-block-builder](https://docs.rs/sc-block-builder/0.8.0/sc_block_builder/index.html) - execution of extrisnics from previous block on block creation
+
+# Shuffling
+Extrinsics are randomly shuffled but still preservers original order per every account - motivation for such algorithm was the fact that following transactions from a given account may depend on other transactions from the same account.
+
+
+Origin order:
+```
+| Alice TX1 | Alice TX2 |  Alice TX3 |  Bob TX1 | Bob TX2 |
+```
+example shuffled order:
+
+```
+| Alice TX1 |  Bob TX1  |  Alice TX2 |  Bob TX2 | Alice TX3 |
+```
+
+As shuffling occurs on block execution not creation, every node needs to calculate the same order and agree on storage state afterwards. That is achieved using Fisher-Yates shuffling algorithm with Xoshiro256++ as PRNG initialized with seed stored in blockchain storage.  Usage of sr25519 key pair together with VRF capabilities guarantees that block producer cannot impact execution order and it's easy to validate if the seed was properly calculated by validators/following nodes (thanks to VRF verification that only requires block author public key). Seed itself is stored into the storage and is publically accessible through dedicated runtime API [RandomSeedApi](https://github.com/mangata-finance/mangata-node/blob/59b8e6d27c76f89cddbad777ffbeafd1d7f86297/pallets/random-seed/runtime-api/src/lib.rs).
+
+PRNG initialization seed for block `N-1` is signature of 'input message' signed with block producer private key where 'input message' is defined as:
+- Seed from block `N-1`
+- babe epoch randomness (changed in every epoch)
+
+There is no way for block producer to manipulate seed value in order to impact execution order of extrinsics - also block producer is not able to calculate seed for a block he is going to author sooner than N-1 block is produced.
+
+Seed value is injected into the block as [InherentData](https://docs.rs/sp-inherents/2.0.0/sp_inherents/struct.InherentData.html) by Babe consensus protocol. Whether block producer finds out it's his turn to create a block new seed is being calculated. Seeds value is validated (VRF verify) by other nodes when new block is being imported - in case of seed calculation violation block will be rejected.
+
+
+Affected crates
+- [sc-block-builder](https://docs.rs/sc-block-builder/0.8.0/sc_block_builder/index.html) - extracting seed value on inherents creation
+- [sc-basic-authorship](https://docs.rs/sc-basic-authorship/0.8.0/sc_basic_authorship/index.html) - shuffling extrinsics before passing them to block builder
+- [sc-consensus-babe](https://docs.rs/sc-basic-authorship/0.8.0/sc_basic_authorship/index.html) - calcuating and injecting shuffling seed value into InherentData, shuffling seed verification
+- [sc-service](https://docs.rs/sc-service/0.8.0/sc_service/index.html) - fetching seed value and extrinsic shuffling for 'following nodes'
+#Tokens
+
+Mangata uses concept of liquidity tokens. One can create a liquidity pool(together with liquidity pool token) using a pair of tokens and then use the tokens of that pool to stake. New currency trait [`MultiTokenCurrency`](https://github.com/mangata-finance/mangata-node/blob/0846c42a7b7fd29e19fd1b30043ddb3b55a8f250/pallets/tokens/src/multi_token_currency.rs#L14) was introduced  and integrated with [staking pallet](https://github.com/mangata-finance/mangata-node/tree/59b8e6d27c76f89cddbad777ffbeafd1d7f86297/pallets/staking). Origin implementation was not sufficient as it only works with single, statically defined currency in our case we require supporting multiple dynamically created currencies.
+
+
+Affected crates
+- [orml-tokens](https://docs.rs/orml-tokens/0.3.1/orml_tokens/index.html) - Introduced new `MultiTokenCurrency` trait, using it as `Balances` replacement
+- [pallet-staking](https://docs.rs/pallet-staking/2.0.0/pallet_staking/index.html) - Alining with `MultiTokenCurrency` trait.
+
+#JS API
+Because of shuffled delayed transaction execution same applies to events triggered by extrinsics. Origin JS API is not sufficient to map extrinsics to triggered events. Link between transaction <-> event is done based on transaction/events order. In Mangata order of extrinsics inside the block differs from their actual execution order. For that reason dedicated async API methods were introduced to provide missing functionality. When transaction is sent:
+1. API waits for notification that it has been included in block N
+2. API fetches block N and N+1
+3. API stores information about list of events produced in block N+1
+3. API reads seed from block N+1 and calculate execution order of extrinsics from block N
+4. API maps events to shuffled extrinsics list
+
+from API client perspective it's as easy as 
+```
+  signAndWaitTx(
+		api.tx.tokens.transfer(to, asset_id, amount),
+        from,
+        nonce
+   ).then( (events) => {
+      events.forEach((e) => {
+          console.log(e.toHuman())
+      });
+  })
+
+```