diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 0c8e0f3511..7855dfde99 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -2,14 +2,26 @@ name: Security audit - daily 'on': + push: + paths: + # Run if workflow changes + - '.github/workflows/audit.yml' + # Run on changed dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # Run if the configuration file changes + - '**/audit.toml' + # Rerun periodicly to pick up new advisories schedule: - cron: '43 05 * * *' + # Run manually + workflow_dispatch: jobs: security_audit: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions-rs/audit-check@v1 + - uses: rustsec/audit-check@v1.4.1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/base_node_binaries.yml b/.github/workflows/base_node_binaries.yml index 87978ffc18..8140f7d85c 100644 --- a/.github/workflows/base_node_binaries.yml +++ b/.github/workflows/base_node_binaries.yml @@ -90,23 +90,7 @@ jobs: if: ${{ startsWith(github.ref, 'refs/tags/v') }} shell: bash run: | - tagnet=${{github.ref_name}} - echo $tagnet - # case match is not RegEx, but wildcards/globs - case "$tagnet" in - v*-pre.*) - TARI_NETWORK=esme - TARI_NETWORK_DIR=testnet - ;; - v*-rc.*) - TARI_NETWORK=nextnet - TARI_NETWORK_DIR=nextnet - ;; - *) - TARI_NETWORK=mainnet - TARI_NETWORK_DIR=mainnet - ;; - esac + source buildtools/multinet_envs.sh ${{github.ref_name}} echo ${TARI_NETWORK} echo ${TARI_NETWORK_DIR} echo "TARI_NETWORK=${TARI_NETWORK}" >> $GITHUB_ENV @@ -129,13 +113,11 @@ jobs: echo "S3DESTOVERRIDE=daily/" >> $GITHUB_ENV - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal components: rustfmt, clippy toolchain: ${{ matrix.builds.rust }} - target: ${{ matrix.builds.target }} - override: true + targets: ${{ matrix.builds.target }} - name: Install Linux dependencies - Ubuntu if: ${{ startsWith(runner.os,'Linux') && matrix.builds.name != 'linux-arm64' }} @@ -189,6 +171,7 @@ jobs: xcrun --show-sdk-path ls -la "/Library/Developer/CommandLineTools/SDKs/" echo "RANDOMX_RS_CMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk" >> $GITHUB_ENV + rustup target add ${{ matrix.builds.target }} - name: Set environment variables - Ubuntu if: startsWith(runner.os,'Linux') @@ -215,14 +198,26 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build rust binaries - uses: actions-rs/cargo@v1 env: + CARGO_UNSTABLE_SPARSE_REGISTRY: true RUSTFLAGS: "-C target_cpu=${{ matrix.builds.target_cpu }}" ROARING_ARCH: "${{ matrix.builds.target_cpu }}" - with: - use-cross: ${{ matrix.builds.cross }} - command: build - args: --release --target ${{ matrix.builds.target }} --features ${{ matrix.builds.features }} ${{ matrix.builds.target_bins }} ${{ matrix.builds.flags }} --locked + shell: bash + run: | + if [ "${{ matrix.builds.cross }}" != "true" ]; then + cargo build --release \ + --target ${{ matrix.builds.target }} \ + --features ${{ matrix.builds.features }} \ + ${{ matrix.builds.target_bins }} \ + ${{ matrix.builds.flags }} --locked + else + cargo install cross + cross build --release \ + --target ${{ matrix.builds.target }} \ + --features ${{ matrix.builds.features }} \ + ${{ matrix.builds.target_bins }} \ + ${{ matrix.builds.flags }} --locked + fi - name: Copy binaries to folder for archiving shell: bash @@ -339,6 +334,13 @@ jobs: echo -e "\nStapling package...${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg\n" xcrun stapler staple -v "${distDirPKG}/${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg" fi + cd ${distDirPKG} + echo "Compute pkg shasum" + ${SHARUN} "${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg" \ + >> "${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg.sha256" + cat "${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg.sha256" + echo "Checksum verification for pkg is " + ${SHARUN} --check "${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg.sha256" - name: Artifact upload for macOS pkg if: startsWith(runner.os,'macOS') @@ -346,7 +348,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: ${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg - path: "${{ env.distDirPKG }}/${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}*.pkg" + path: "${{ env.distDirPKG }}/${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}*.pkg*" # unlike inno script studio, iscc.exe doesn't run the [precompile] step generate_config.bat - name: Build the Windows installer @@ -369,19 +371,17 @@ jobs: run: | echo "Archive ${{ env.BINFILE }} too ${{ env.BINFILE }}.zip" cd "$GITHUB_WORKSPACE${{ env.TBN_DIST }}" - #zip -j "${{ env.BINFILE }}.zip" * + echo "Compute files shasum" + ${SHARUN} * >> "${{ env.BINFILE }}.sha256" + cat "${{ env.BINFILE }}.sha256" + echo "Checksum verification for files is " + ${SHARUN} --check "${{ env.BINFILE }}.sha256" 7z a "${{ env.BINFILE }}.zip" * - echo "Compute shasum" + echo "Compute archive shasum" ${SHARUN} "${{ env.BINFILE }}.zip" >> "${{ env.BINFILE }}.zip.sha256" cat "${{ env.BINFILE }}.zip.sha256" - echo "Verifications is " + echo "Checkum verification archive is " ${SHARUN} --check "${{ env.BINFILE }}.zip.sha256" - if [ -f "${{ env.distDirPKG }}/${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg" ]; then - echo "Add PKG to $GITHUB_WORKSPACE${{ env.TBN_DIST }} " - cp -v "${{ env.distDirPKG }}/${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg" "$GITHUB_WORKSPACE${{ env.TBN_DIST }}" - ${SHARUN} "${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg" >> "${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg.sha256" - ${SHARUN} --check "${{ env.TBN_FILENAME }}-${{ matrix.builds.name }}-${{ env.VERSION }}.pkg.sha256" - fi - name: Artifact upload for Archive uses: actions/upload-artifact@v3 @@ -389,14 +389,18 @@ jobs: name: ${{ env.TBN_FILENAME }}_archive-${{ matrix.builds.name }} path: "${{ github.workspace }}${{ env.TBN_DIST }}/${{ env.BINFILE }}.zip*" - - name: Prep miner for upload + - name: Prep Miner for upload shell: bash run: | cd "${{ github.workspace }}${{ env.TBN_DIST }}" cp -v "tari_miner${{ env.TBN_EXT}}" \ "tari_miner-${{ matrix.builds.name }}${{ env.TBN_EXT}}" + echo "Compute miner shasum" ${SHARUN} "tari_miner-${{ matrix.builds.name }}${{ env.TBN_EXT}}" \ >> "tari_miner-${{ matrix.builds.name }}${{ env.TBN_EXT}}.sha256" + cat "tari_miner-${{ matrix.builds.name }}${{ env.TBN_EXT}}.sha256" + echo "Checksum verification for miner is " + ${SHARUN} --check "tari_miner-${{ matrix.builds.name }}${{ env.TBN_EXT}}.sha256" - name: Artifact upload for Miner uses: actions/upload-artifact@v3 @@ -404,7 +408,7 @@ jobs: name: tari_miner-${{ matrix.builds.name }} path: "${{ github.workspace }}${{ env.TBN_DIST }}/tari_miner-${{ matrix.builds.name }}${{ env.TBN_EXT}}*" - - name: Prep diag-utils for upload + - name: Prep diag-utils archive for upload continue-on-error: true shell: bash run: | @@ -413,15 +417,26 @@ jobs: # Find RandomX built tools for testing find "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/" \ -name "randomx-*${{ env.TBN_EXT}}" -type f -perm -+x -exec cp -v {} . \; + echo "Compute diag utils shasum" ${SHARUN} * \ >> "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256" + cat "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256" + echo "Checksum verification for diag utils is " + ${SHARUN} --check "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256" + 7z a "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip" * + echo "Compute diag utils archive shasum" + ${SHARUN} "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip" \ + >> "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256" + cat "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256" + echo "Checksum verification for diag utils archive is " + ${SHARUN} --check "${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256" - name: Artifact upload for diag-utils continue-on-error: true uses: actions/upload-artifact@v3 with: name: ${{ env.TBN_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }} - path: "${{ github.workspace }}${{ env.TBN_DIST }}/diag-utils/*" + path: "${{ github.workspace }}${{ env.TBN_DIST }}/diag-utils/*.zip*" - name: Sync dist to S3 - Bash continue-on-error: true # Don't break if s3 upload fails @@ -498,7 +513,6 @@ jobs: with: artifacts: "tari_*/**/*" token: ${{ secrets.GITHUB_TOKEN }} - #bodyFile: changelog.md prerelease: true draft: true allowUpdates: true diff --git a/.github/workflows/build_dockers_workflow.yml b/.github/workflows/build_dockers_workflow.yml index 4eca2f60f3..1667095c8e 100644 --- a/.github/workflows/build_dockers_workflow.yml +++ b/.github/workflows/build_dockers_workflow.yml @@ -116,17 +116,7 @@ jobs: if: ${{ startsWith(github.ref, 'refs/tags/v') }} shell: bash run: | - tagnet=${{github.ref_name}} - echo $tagnet - # case match is not RegEx, but wildcards/globs - case "$tagnet" in - v*-pre.*) TARI_NETWORK=esme - ;; - v*-rc.*) TARI_NETWORK=nextnet - ;; - *) TARI_NETWORK=mainnet - ;; - esac + source tari/buildtools/multinet_envs.sh ${{github.ref_name}} echo ${TARI_NETWORK} echo "TARI_NETWORK=${TARI_NETWORK}" >> $GITHUB_ENV diff --git a/.github/workflows/build_libwallets.yml b/.github/workflows/build_libwallets.yml index d4e7de17b2..3fa0c10aec 100644 --- a/.github/workflows/build_libwallets.yml +++ b/.github/workflows/build_libwallets.yml @@ -70,9 +70,9 @@ jobs: working-directory: libwallets run: | ls -alhtR - find . -name "*.sha256sums" -type f -print | xargs cat >> libwallets.txt.sha256sums-verify - cat libwallets.txt.sha256sums-verify - sha256sum -c libwallets.txt.sha256sums-verify + find . -name "*.sha256" -type f -print | xargs cat >> libwallets.txt.sha256-verify + cat libwallets.txt.sha256-verify + sha256sum -c libwallets.txt.sha256-verify - name: Sync to S3 on tag if: ${{ startsWith(github.ref, 'refs/tags/v') }} @@ -98,18 +98,12 @@ jobs: with: path: libwallets -# - name: Setup changelog -# shell: bash -# working-directory: libwallets -# run: | -# find . -name changelog.md -type f -exec cp -vf {} "$GITHUB_WORKSPACE/" \; - - - name: Update sha256sums for top level paths + - name: Update sha256 for top level paths shell: bash working-directory: libwallets run: | ls -alht - find . -name "libtari_wallet_ffi.*.sha256sums" -type f \ + find . -name "libtari_wallet_ffi.*.sha256" -type f \ -exec sed -i -e "s/libwallet-.*\///g" '{}' \; ls -alht @@ -119,12 +113,11 @@ jobs: run: | ls -alht if [ -d libwallet-ios-xcframework ]; then - #zip -j libtari_wallet_ffi.ios-xcframework.zip libwallet-ios-xcframework/* 7z a libtari_wallet_ffi.ios-xcframework.zip libwallet-ios-xcframework/* rm -fr libwallet-ios-xcframework/* shasum -a 256 \ "libtari_wallet_ffi.ios-xcframework.zip" \ - > "libtari_wallet_ffi.ios-xcframework.zip.sha256sums" + > "libtari_wallet_ffi.ios-xcframework.zip.sha256" fi ls -alht @@ -133,7 +126,6 @@ jobs: with: artifacts: "libwallet*/**/*" token: ${{ secrets.GITHUB_TOKEN }} - #bodyFile: "changelog.md" prerelease: true draft: true allowUpdates: true diff --git a/.github/workflows/build_libwallets_workflow.yml b/.github/workflows/build_libwallets_workflow.yml index 2f4e3c0bb0..b0f5162333 100644 --- a/.github/workflows/build_libwallets_workflow.yml +++ b/.github/workflows/build_libwallets_workflow.yml @@ -15,6 +15,10 @@ name: Build libwallet - workflow_call/on-demand description: 'Rust toolchain' default: 'stable' +env: + CARGO_UNSTABLE_SPARSE_REGISTRY: true + TARI_NETWORK_CHANGELOG: "development" + jobs: android_build: name: Build Android @@ -37,43 +41,30 @@ jobs: if: ${{ startsWith(github.ref, 'refs/tags/v') }} shell: bash run: | - tagnet=${{github.ref_name}} - echo $tagnet - # case match is not RegEx, but wildcards/globs - case "$tagnet" in - v*-pre.*) TARI_NETWORK=esme - ;; - v*-rc.*) TARI_NETWORK=nextnet - ;; - *) TARI_NETWORK=mainnet - ;; - esac + source buildtools/multinet_envs.sh ${{github.ref_name}} echo ${TARI_NETWORK} + echo ${TARI_NETWORK_CHANGELOG} echo "TARI_NETWORK=${TARI_NETWORK}" >> $GITHUB_ENV + echo "TARI_NETWORK_CHANGELOG=${TARI_NETWORK_CHANGELOG}" >> $GITHUB_ENV - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal components: rustfmt, clippy toolchain: ${{ inputs.toolchain }} - target: ${{ matrix.build }} - override: true + targets: ${{ matrix.build }} - name: Cache cargo files and outputs uses: Swatinem/rust-cache@v2 - name: Build libwallet libraries - uses: actions-rs/cargo@v1 env: CFLAGS: "-DMDB_USE_ROBUST=0" - with: - use-cross: true - command: build - args: > - --lib - --release - --package tari_wallet_ffi + shell: bash + run: | + cargo install cross + cross build --lib --release \ + --package tari_wallet_ffi \ --target=${{ matrix.build }} - name: Prepare libwallet libraries @@ -87,12 +78,16 @@ jobs: cd "${{ runner.temp }}/libwallet-android-${target_platform}" cp -v "$GITHUB_WORKSPACE/target/${{ matrix.build }}/release/libtari_wallet_ffi.a" "libtari_wallet_ffi.android_${target_platform}.a" cp -v "$GITHUB_WORKSPACE/base_layer/wallet_ffi/wallet.h" libtari_wallet_ffi.h - cp -v "$GITHUB_WORKSPACE/changelog.md" . + if [ -f "$GITHUB_WORKSPACE/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md" ]; then + cp -v "$GITHUB_WORKSPACE/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md" . + TARI_NETWORK_CHANGELOG_FILE=libwallet-android-${target_platform}/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md + echo ${TARI_NETWORK_CHANGELOG_FILE} + fi cd .. shasum -a 256 \ "libwallet-android-${target_platform}/libtari_wallet_ffi.android_${target_platform}.a" \ - "libwallet-android-${target_platform}/libtari_wallet_ffi.h" \ - > "libwallet-android-${target_platform}/libtari_wallet_ffi.android_${target_platform}.sha256sums" + "libwallet-android-${target_platform}/libtari_wallet_ffi.h" "${TARI_NETWORK_CHANGELOG_FILE}" \ + > "libwallet-android-${target_platform}/libtari_wallet_ffi.android_${target_platform}.sha256" ls -alht "${{ runner.temp }}/libwallet-android-${target_platform}" - name: Upload artifacts @@ -123,28 +118,18 @@ jobs: if: ${{ startsWith(github.ref, 'refs/tags/v') }} shell: bash run: | - tagnet=${{github.ref_name}} - echo $tagnet - # case match is not RegEx, but wildcards/globs - case "$tagnet" in - v*-pre.*) TARI_NETWORK=esme - ;; - v*-rc.*) TARI_NETWORK=nextnet - ;; - *) TARI_NETWORK=mainnet - ;; - esac + source buildtools/multinet_envs.sh ${{github.ref_name}} echo ${TARI_NETWORK} + echo ${TARI_NETWORK_CHANGELOG} echo "TARI_NETWORK=${TARI_NETWORK}" >> $GITHUB_ENV + echo "TARI_NETWORK_CHANGELOG=${TARI_NETWORK_CHANGELOG}" >> $GITHUB_ENV - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal components: rustfmt, clippy toolchain: ${{ inputs.toolchain }} - target: ${{ matrix.build }} - override: true + targets: ${{ matrix.build }} - name: Install macOS dependencies run: brew install cmake coreutils @@ -153,14 +138,11 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build libwallet libraries - uses: actions-rs/cargo@v1 - with: - use-cross: false - command: build - args: > - --lib - --release - --package tari_wallet_ffi + shell: bash + run: | + rustup target add ${{ matrix.build }} + cargo build --lib --release \ + --package tari_wallet_ffi \ --target=${{ matrix.build }} - name: Prepare libwallet libraries @@ -174,12 +156,17 @@ jobs: cd "${{ runner.temp }}/libwallet-ios-${target_platform}" cp -v "$GITHUB_WORKSPACE/target/${{ matrix.build }}/release/libtari_wallet_ffi.a" "libtari_wallet_ffi.ios_${target_platform}.a" cp -v "$GITHUB_WORKSPACE/base_layer/wallet_ffi/wallet.h" libtari_wallet_ffi.h - cp -v "$GITHUB_WORKSPACE/changelog.md" . + if [ -f "$GITHUB_WORKSPACE/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md" ]; then + cp -v "$GITHUB_WORKSPACE/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md" . + TARI_NETWORK_CHANGELOG_FILE=libwallet-ios-${target_platform}/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md + echo ${TARI_NETWORK_CHANGELOG_FILE} + fi + cd .. shasum -a 256 \ "libwallet-ios-${target_platform}/libtari_wallet_ffi.ios_${target_platform}.a" \ - "libwallet-ios-${target_platform}/libtari_wallet_ffi.h" \ - > "libwallet-ios-${target_platform}/libtari_wallet_ffi.ios_${target_platform}.sha256sums" + "libwallet-ios-${target_platform}/libtari_wallet_ffi.h" "${TARI_NETWORK_CHANGELOG_FILE}" \ + > "libwallet-ios-${target_platform}/libtari_wallet_ffi.ios_${target_platform}.sha256" ls -alht "${{ runner.temp }}/libwallet-ios-${target_platform}" - name: Upload artifacts @@ -192,6 +179,7 @@ jobs: name: Assemble iOS universal if: ${{ inputs.build_ios == 'true' }} needs: ios_build + strategy: fail-fast: false @@ -211,29 +199,45 @@ jobs: working-directory: libwallets run: | ls -alhtR - find . -name "*.sha256sums" -type f -print | xargs cat >> libwallets.txt.sha256sums-verify - cat libwallets.txt.sha256sums-verify - sha256sum -c libwallets.txt.sha256sums-verify + find . -name "*.sha256" -type f -print | xargs cat >> libwallets.txt.sha256-verify + cat libwallets.txt.sha256-verify + sha256sum -c libwallets.txt.sha256-verify - name: Assemble iOS universal libwallet shell: bash working-directory: libwallets run: | - ls -alht + ls -alhtR mkdir libwallet-ios-universal cp -v "libwallet-ios-x86_64/libtari_wallet_ffi.h" \ libwallet-ios-universal/ - cp -v "libwallet-ios-x86_64/changelog.md" \ - libwallet-ios-universal/ + echo "Check for changelog" + if [ -f libwallet-ios-x86_64/changelog-*.md ]; then + echo "Changelog found" + envChangelogFull=$(ls libwallet-ios-x86_64/changelog-*.md) + echo ${envChangelogFull} + # Strip suffix + #envChangelog=${envChangelogFull::-3} + envChangelog=${envChangelogFull:0:${#envChangelogFull}-3} + echo ${envChangelog} + # Strip prefix + TARI_NETWORK_CHANGELOG=${envChangelog##*/changelog-} + echo ${TARI_NETWORK_CHANGELOG} + cp -v "libwallet-ios-x86_64/changelog-${TARI_NETWORK_CHANGELOG}.md" libwallet-ios-universal/ + TARI_NETWORK_CHANGELOG_FILE=libwallet-ios-universal/changelog-${TARI_NETWORK_CHANGELOG}.md + echo ${TARI_NETWORK_CHANGELOG_FILE} + else + echo "No changelog found" + fi lipo -create \ "libwallet-ios-x86_64/libtari_wallet_ffi.ios_x86_64.a" \ "libwallet-ios-aarch64/libtari_wallet_ffi.ios_aarch64.a" \ -output "libwallet-ios-universal/libtari_wallet_ffi.ios_universal.a" shasum -a 256 \ "libwallet-ios-universal/libtari_wallet_ffi.ios_universal.a" \ - "libwallet-ios-universal/libtari_wallet_ffi.h" \ - > "libwallet-ios-universal/libtari_wallet_ffi.ios_universal.sha256sums" - ls -alht + "libwallet-ios-universal/libtari_wallet_ffi.h" "${TARI_NETWORK_CHANGELOG_FILE}" \ + > "libwallet-ios-universal/libtari_wallet_ffi.ios_universal.sha256" + ls -alhtR - name: Upload iOS universal libwallet artifacts uses: actions/upload-artifact@v3 @@ -245,15 +249,31 @@ jobs: shell: bash working-directory: libwallets run: | - ls -alht + ls -alhtR mkdir libwallet-ios-universal-sim lipo -create \ "libwallet-ios-x86_64/libtari_wallet_ffi.ios_x86_64.a" \ "libwallet-ios-aarch64-sim/libtari_wallet_ffi.ios_aarch64-sim.a" \ -output "libwallet-ios-universal-sim/libtari_wallet_ffi.ios_universal-sim.a" mkdir libwallet-ios-xcframework - cp -v "libwallet-ios-x86_64/changelog.md" \ - libwallet-ios-xcframework/ + echo "Check for changelog" + if [ -f libwallet-ios-x86_64/changelog-*.md ]; then + echo "Changelog found" + envChangelogFull=$(ls libwallet-ios-x86_64/changelog-*.md) + echo ${envChangelogFull} + # Strip suffix + #envChangelog=${envChangelogFull::-3} + envChangelog=${envChangelogFull:0:${#envChangelogFull}-3} + echo ${envChangelog} + # Strip prefix + TARI_NETWORK_CHANGELOG=${envChangelog##*/changelog-} + echo ${TARI_NETWORK_CHANGELOG} + cp -v "libwallet-ios-x86_64/changelog-${TARI_NETWORK_CHANGELOG}.md" libwallet-ios-xcframework/ + TARI_NETWORK_CHANGELOG_FILE=libwallet-ios-xcframework/changelog-${TARI_NETWORK_CHANGELOG}.md + echo ${TARI_NETWORK_CHANGELOG_FILE} + else + echo "No changelog found" + fi xcodebuild -create-xcframework \ -library "libwallet-ios-universal-sim/libtari_wallet_ffi.ios_universal-sim.a" \ -headers "libwallet-ios-x86_64/libtari_wallet_ffi.h" \ @@ -266,8 +286,9 @@ jobs: "libwallet-ios-xcframework/libtari_wallet_ffi_ios.xcframework/ios-arm64/libtari_wallet_ffi.ios_aarch64.a" \ "libwallet-ios-xcframework/libtari_wallet_ffi_ios.xcframework/ios-arm64_x86_64-simulator/Headers" \ "libwallet-ios-xcframework/libtari_wallet_ffi_ios.xcframework/ios-arm64_x86_64-simulator/libtari_wallet_ffi.ios_universal-sim.a" \ - > "libwallet-ios-xcframework/libtari_wallet_ffi.ios_xcframework.sha256sums" - ls -alht + "${TARI_NETWORK_CHANGELOG_FILE}" \ + > "libwallet-ios-xcframework/libtari_wallet_ffi.ios_xcframework.sha256" + ls -alhtR - name: Upload iOS xcframework libwallet artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b91927e69..783a82bab8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ --- name: CI -"on": +'on': workflow_dispatch: push: branches: @@ -13,12 +13,13 @@ name: CI - opened - reopened - synchronize - merge_group: + merge_group: env: toolchain: nightly-2022-11-03 CARGO_HTTP_MULTIPLEXING: false CARGO_TERM_COLOR: always + CARGO_UNSTABLE_SPARSE_REGISTRY: true PROTOC: protoc TERM: unkown @@ -29,13 +30,12 @@ concurrency: jobs: clippy: name: clippy - #runs-on: [ self-hosted, ubuntu18.04-high-cpu ] runs-on: [ubuntu-20.04] steps: - name: checkout uses: actions/checkout@v3 - name: toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.toolchain }} components: clippy, rustfmt @@ -43,77 +43,74 @@ jobs: run: | sudo apt-get update sudo bash scripts/install_ubuntu_dependencies.sh - - name: cargo fmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check + - name: cargo format + run: cargo fmt --all -- --check - name: Install cargo-lints - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-lints + run: cargo install cargo-lints - name: Clippy check (with lints) - uses: actions-rs/cargo@v1 + run: cargo lints clippy --all-targets --all-features + machete: + name: machete + runs-on: [ubuntu-20.04] + steps: + - name: checkout + uses: actions/checkout@v3 + - name: toolchain + uses: dtolnay/rust-toolchain@master with: - command: lints - args: clippy --all-targets --all-features - build: - name: cargo check - runs-on: [self-hosted, ubuntu18.04-high-cpu] + toolchain: ${{ env.toolchain }} + components: clippy, rustfmt + - name: ubuntu dependencies + run: | + sudo apt-get update + sudo bash scripts/install_ubuntu_dependencies.sh + - name: cargo machete + run: | + cargo install cargo-machete + cargo machete + build-nightly: + name: cargo check with nightly + runs-on: [self-hosted, ubuntu-high-cpu] steps: - name: checkout uses: actions/checkout@v3 - name: toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.toolchain }} - profile: minimal - override: true - name: ubuntu dependencies run: | sudo apt-get update sudo bash scripts/install_ubuntu_dependencies.sh + - name: rustup show + run: rustup show - name: cargo check - uses: actions-rs/cargo@v1 - with: - command: check - args: --release --all-targets + run: cargo check --release --all-targets --locked - name: cargo check ffi separately - uses: actions-rs/cargo@v1 - with: - command: check - args: --release --package tari_wallet_ffi + run: cargo check --release --package tari_wallet_ffi --locked build-stable: - name: cargo check stable - runs-on: [self-hosted, ubuntu18.04-high-cpu] + name: cargo check with stable + runs-on: [self-hosted, ubuntu-high-cpu] steps: - name: checkout uses: actions/checkout@v3 + - name: rust-toolchain.toml override by removing + run: rm -f rust-toolchain.toml - name: toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: toolchain: stable - profile: minimal - override: true - uses: Swatinem/rust-cache@v2 - name: ubuntu dependencies run: | sudo apt-get update sudo bash scripts/install_ubuntu_dependencies.sh - name: rustup show - run: | - rustup show + run: rustup show - name: cargo check - uses: actions-rs/cargo@v1 - with: - command: check - args: --release --all-targets --workspace --exclude tari_integration_tests --locked + run: cargo check --release --all-targets --workspace --exclude tari_integration_tests --locked - name: cargo check ffi separately - uses: actions-rs/cargo@v1 - with: - command: check - args: --release --package tari_wallet_ffi + run: cargo check --release --package tari_wallet_ffi --locked licenses: name: file licenses runs-on: [ubuntu-20.04] @@ -129,12 +126,15 @@ jobs: run: ./scripts/file_license_check.sh test: name: test - runs-on: [self-hosted, ubuntu18.04-high-cpu] + runs-on: [self-hosted, ubuntu-high-cpu] + permissions: + checks: write + pull-requests: write steps: - name: checkout uses: actions/checkout@v3 - name: toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.toolchain }} - name: ubuntu dependencies @@ -143,20 +143,21 @@ jobs: sudo bash scripts/install_ubuntu_dependencies.sh - name: test key manager wasm run: | - npm install -g wasm-pack@0.10.3 cd base_layer/key_manager rustup target add wasm32-unknown-unknown make test + - name: Install cargo-nextest + run: cargo install cargo-nextest --locked - name: cargo test compile - uses: actions-rs/cargo@v1 - with: - command: test - args: --no-run --locked --all-features --release + run: cargo test --no-run --locked --all-features --release - name: cargo test - uses: actions-rs/cargo@v1 + run: cargo nextest run --all-features --release -E "not package(tari_integration_tests)" --profile ci + - name: upload artifact + uses: actions/upload-artifact@v3 # upload test results as artifact + if: always() with: - command: test - args: -v --all-features --release --workspace --exclude tari_integration_tests + name: test-results + path: ${{ github.workspace }}/target/nextest/ci/junit.xml # Allows other workflows to know the PR number artifacts: @@ -174,3 +175,14 @@ jobs: with: name: pr_num path: ./pr_num.txt + +# needed for test results + event_file: + name: "Upload Event File for Test Results" + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: Event File + path: ${{ github.event_path }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6a4d4d148e..0ca830dedd 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,7 +17,7 @@ concurrency: jobs: coverage: name: test and generate coverage - runs-on: [ self-hosted, ubuntu18.04-high-mem ] + runs-on: [ self-hosted, ubuntu-high-mem ] steps: - name: checkout source code uses: actions/checkout@v3 @@ -27,10 +27,9 @@ jobs: sudo apt-get update sudo bash scripts/install_ubuntu_dependencies.sh - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@master with: toolchain: nightly - override: true components: llvm-tools-preview - name: cache cargo files and outputs @@ -38,14 +37,13 @@ jobs: with: cache-on-failure: true - - uses: actions-rs/cargo@v1 + - name: cargo test env: RUSTFLAGS: "-C instrument-coverage" RUSTDOCFLAGS: "-C instrument-coverage" LLVM_PROFILE_FILE: "coverage_data-%p-%m.profraw" - with: - command: test - args: --all-features --no-fail-fast --workspace --exclude tari_integration_tests + CARGO_UNSTABLE_SPARSE_REGISTRY: true + run: cargo test --all-features --no-fail-fast --workspace --exclude tari_integration_tests - name: prepare coverage data env: diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 7030362c36..6760a0ef2e 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1,13 +1,13 @@ --- name: Integration tests -'on': +"on": pull_request: types: - opened - reopened - synchronize - merge_group: + merge_group: push: paths-ignore: - '**/*.md' @@ -39,7 +39,7 @@ concurrency: jobs: base_layer: name: Cucumber tests / Base Layer - runs-on: [self-hosted, ubuntu18.04-high-cpu] + runs-on: [self-hosted, ubuntu-high-cpu] steps: - name: checkout uses: actions/checkout@v3 @@ -64,12 +64,10 @@ jobs: fi - name: Setup rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal components: rustfmt, clippy toolchain: ${{ env.toolchain }} - override: true - name: Install ubuntu dependencies shell: bash @@ -81,30 +79,32 @@ jobs: uses: Swatinem/rust-cache@v2 - name: cargo test compile - uses: actions-rs/cargo@v1 - with: - command: test - args: --no-run --locked --all-features --release ${{ env.TARGET_BINS }} + run: cargo test --no-run --locked --all-features --release ${{ env.TARGET_BINS }} - name: Run ${{ env.CI_PROFILE }} integration tests for binaries if: ${{ env.CI_BINS == 'true' }} timeout-minutes: 90 - uses: actions-rs/cargo@v1 - with: - command: test - args: > - --test cucumber - -v - --all-features - --release - --package tari_integration_tests - -- -t "${{ env.CI_PROFILE }} and (not @wallet-ffi) and (not @broken)" - -c 5 + run: | + cargo test \ + --test cucumber \ + -v \ + --all-features \ + --release \ + --package tari_integration_tests \ + -- -t "${{ env.CI_PROFILE }} and (not @wallet-ffi) and (not @chat-ffi) and (not @broken)" \ + -c 5 \ --retry 2 + - name: upload artifact + uses: actions/upload-artifact@v3 # upload test results as artifact + if: always() + with: + name: junit-cucumber + path: ${{ github.workspace }}/integration_tests/cucumber-output-junit.xml + ffi: name: Cucumber tests / FFI - runs-on: [self-hosted, ubuntu18.04-high-cpu] + runs-on: [self-hosted, ubuntu-high-cpu] steps: - name: checkout uses: actions/checkout@v3 @@ -130,12 +130,10 @@ jobs: - name: Setup rust toolchain if: ${{ env.CI_FFI == 'true' }} - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal components: rustfmt, clippy toolchain: ${{ env.toolchain }} - override: true - name: Install ubuntu dependencies if: ${{ env.CI_FFI == 'true' }} @@ -150,23 +148,36 @@ jobs: - name: cargo test compile if: ${{ env.CI_FFI == 'true' }} - uses: actions-rs/cargo@v1 - with: - command: test - args: --no-run --locked --all-features --release ${{ env.TARGET_BINS }} + run: cargo test --no-run --locked --all-features --release ${{ env.TARGET_BINS }} - name: Run ${{ env.CI_PROFILE }} integration tests for ffi if: ${{ env.CI_FFI == 'true' }} timeout-minutes: 90 - uses: actions-rs/cargo@v1 - with: - command: test - args: > - --test cucumber - -v - --all-features - --release - --package tari_integration_tests - -- -t "@wallet-ffi and ${{ env.CI_PROFILE }} and (not @broken)" - -c 1 + run: | + cargo test \ + --test cucumber \ + -v \ + --all-features \ + --release \ + --package tari_integration_tests \ + -- -t "(@wallet-ffi or @chat-ffi) and ${{ env.CI_PROFILE }} and (not @broken)" \ + -c 1 \ --retry 2 + + - name: upload artifact + uses: actions/upload-artifact@v3 # upload test results as artifact + if: always() + with: + name: junit-ffi-cucumber + path: ${{ github.workspace }}/integration_tests/cucumber-output-junit.xml + + # needed for test results + event_file: + name: "Upload Event File for Test Results" + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: Event File + path: ${{ github.event_path }} diff --git a/.github/workflows/publish_test_results_ci.yml b/.github/workflows/publish_test_results_ci.yml new file mode 100644 index 0000000000..534f25a31d --- /dev/null +++ b/.github/workflows/publish_test_results_ci.yml @@ -0,0 +1,62 @@ +name: Test Results for CI + +on: + workflow_run: + workflows: ["CI", "Integration Tests"] + types: + - completed +permissions: {} + +jobs: + test-results: + name: Test Results + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion != 'skipped' + + permissions: + checks: write + + # needed unless run with comment_mode: off + pull-requests: write + + # required by download step to access artifacts API + actions: read + + steps: + - name: Download and Extract Artifacts + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + run: | + mkdir -p artifacts && cd artifacts + + artifacts_url=${{ github.event.workflow_run.artifacts_url }} + + gh api --paginate "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact + do + IFS=$'\t' read name url <<< "$artifact" + gh api $url > "$name.zip" + unzip -d "$name" "$name.zip" + done + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + commit: ${{ github.event.workflow_run.head_sha }} + event_file: artifacts/Event File/event.json + check_name: "Test Results (${{ github.event.workflow_run.name || github.event_name }})" + event_name: ${{ github.event.workflow_run.event }} + files: "artifacts/**/*.xml" + + - name: Publish Test Report 2 + uses: mikepenz/action-junit-report@v3 + if: always() + with: + report_paths: "artifacts/**/*.xml" + + - name: Publish Test Report 3 + uses: dorny/test-reporter@v1 + with: + artifact: test-results # artifact name + name: JEST Tests # Name of the check run which will be created + path: "artifacts/**/*.xml" # Path to test results + reporter: jest-junit # Format of test results diff --git a/buildtools/multinet_envs.sh b/buildtools/multinet_envs.sh new file mode 100644 index 0000000000..1650ba9709 --- /dev/null +++ b/buildtools/multinet_envs.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# setup envs based on tag passed +tagnet=$1 +echo $tagnet +# case match is not RegEx, but wildcards/globs +case "$tagnet" in +v*-pre.*) + echo "esme" + export TARI_NETWORK=esme + export TARI_NETWORK_DIR=testnet + ;; +v*-rc.*) + echo "nextnet" + export TARI_NETWORK=nextnet + export TARI_NETWORK_DIR=nextnet + ;; +*) + echo "mainnet" + export TARI_NETWORK=mainnet + export TARI_NETWORK_DIR=mainnet + ;; +esac