From 9435066e6e94329c5e3b729803dad50b82148567 Mon Sep 17 00:00:00 2001 From: "C.Lee Taylor" <47312074+leet4tari@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:08:20 +0200 Subject: [PATCH] ci(fix): update scripts for ci workflows (#3) Description Updated some of the workflow use dscripts Motivation and Context Improve CI How Has This Been Tested? Not What process can a PR reviewer use to test or verify this change? Not yet Breaking Changes --- - [x] None - [ ] Requires data directory on base node to be deleted - [ ] Requires hard fork - [ ] Other - Please specify --- .github/workflows/build_binaries.json | 14 +- .github/workflows/build_binaries.yml | 125 +------------ .license.ignore | 0 scripts/cross_compile_tooling.sh | 59 +++++++ scripts/cross_compile_ubuntu_18-pre-build.sh | 166 ++++++++++++++++++ scripts/file_license_check.sh | 40 +++++ ...stall_ubuntu_dependencies-cross_compile.sh | 21 +++ .../install_ubuntu_dependencies-rust-arm64.sh | 10 ++ scripts/install_ubuntu_dependencies-rust.sh | 10 ++ scripts/install_ubuntu_dependencies.sh | 27 +++ 10 files changed, 347 insertions(+), 125 deletions(-) create mode 100644 .license.ignore create mode 100755 scripts/cross_compile_tooling.sh create mode 100755 scripts/cross_compile_ubuntu_18-pre-build.sh create mode 100755 scripts/file_license_check.sh create mode 100755 scripts/install_ubuntu_dependencies-cross_compile.sh create mode 100755 scripts/install_ubuntu_dependencies-rust-arm64.sh create mode 100755 scripts/install_ubuntu_dependencies-rust.sh create mode 100755 scripts/install_ubuntu_dependencies.sh diff --git a/.github/workflows/build_binaries.json b/.github/workflows/build_binaries.json index 943e5110..9bf9a4a1 100644 --- a/.github/workflows/build_binaries.json +++ b/.github/workflows/build_binaries.json @@ -13,8 +13,8 @@ "rust": "stable", "target": "aarch64-unknown-linux-gnu", "cross": true, - "flags": "--features libtor --workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests", - "build_metric": true + "build_enabled": true, + "best_effort": true }, { "name": "linux-riscv64", @@ -22,7 +22,6 @@ "rust": "stable", "target": "riscv64gc-unknown-linux-gnu", "cross": true, - "flags": "--workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests", "build_enabled": true, "best_effort": true }, @@ -46,8 +45,7 @@ "rust": "stable", "target": "x86_64-pc-windows-msvc", "cross": false, - "features": "safe", - "flags": "--workspace --exclude tari_libtor" + "features": "safe" }, { "name": "windows-arm64", @@ -55,9 +53,7 @@ "rust": "stable", "target": "aarch64-pc-windows-msvc", "cross": false, - "features": "safe", - "target_bins": "minotari_node, minotari_console_wallet, minotari_merge_mining_proxy, minotari_miner", - "flags": "--workspace --exclude tari_libtor", - "build_enabled": false + "build_enabled": true, + "best_effort": true } ] diff --git a/.github/workflows/build_binaries.yml b/.github/workflows/build_binaries.yml index 6bd01edb..1cff433d 100644 --- a/.github/workflows/build_binaries.yml +++ b/.github/workflows/build_binaries.yml @@ -18,14 +18,12 @@ name: Build Matrix of Binaries default: "development-tag" env: - TS_FILENAME: "tari_suite" + TS_FILENAME: "sha_p2pool" TS_BUNDLE_ID_BASE: "com.tarilabs" TS_SIG_FN: "sha256-unsigned.txt" ## Must be a JSon string - TS_FILES: '["minotari_node","minotari_console_wallet","minotari_miner","minotari_merge_mining_proxy"]' - TS_FEATURES: "default, safe" - TS_LIBRARIES: "minotari_mining_helper_ffi" - TARI_NETWORK_DIR: testnet + TS_FILES: '["sha_p2pool"]' + TS_FEATURES: "default" toolchain: nightly-2024-03-01 matrix-json-file: ".github/workflows/build_binaries.json" CARGO_HTTP_MULTIPLEXING: false @@ -93,7 +91,6 @@ jobs: needs: matrix-prep continue-on-error: ${{ matrix.builds.best_effort || false }} outputs: - TARI_NETWORK_DIR: ${{ steps.set-tari-network.outputs.TARI_NETWORK_DIR }} TARI_VERSION: ${{ steps.set-tari-vars.outputs.TARI_VERSION }} VSHA_SHORT: ${{ steps.set-tari-vars.outputs.VSHA_SHORT }} strategy: @@ -105,26 +102,6 @@ jobs: steps: - name: Checkout source code uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Declare TestNet for tags - id: set-tari-network - # Don't forget to comment out the below if, when force testing with GHA_NETWORK - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - env: - GHA_NETWORK: ${{ github.ref_name }} - # GHA_NETWORK: "v1.0.0-rc.4" - shell: bash - run: | - source buildtools/multinet_envs.sh ${{ env.GHA_NETWORK }} - echo ${TARI_NETWORK} - echo ${TARI_TARGET_NETWORK} - echo ${TARI_NETWORK_DIR} - echo "TARI_NETWORK=${TARI_NETWORK}" >> $GITHUB_ENV - echo "TARI_TARGET_NETWORK=${TARI_TARGET_NETWORK}" >> $GITHUB_ENV - echo "TARI_NETWORK_DIR=${TARI_NETWORK_DIR}" >> $GITHUB_ENV - echo "TARI_NETWORK_DIR=${TARI_NETWORK_DIR}" >> $GITHUB_OUTPUT - name: Declare Global Variables 4 GHA ${{ github.event_name }} id: set-tari-vars @@ -136,7 +113,7 @@ jobs: echo "VSHA_SHORT=${VSHA_SHORT}" >> $GITHUB_OUTPUT TARI_VERSION=$(awk -F ' = ' '$1 ~ /^version/ \ { gsub(/["]/, "", $2); printf("%s",$2) }' \ - "$GITHUB_WORKSPACE/applications/minotari_node/Cargo.toml") + "$GITHUB_WORKSPACE/Cargo.toml") echo "TARI_VERSION=${TARI_VERSION}" >> $GITHUB_ENV echo "TARI_VERSION=${TARI_VERSION}" >> $GITHUB_OUTPUT if [[ "${{ matrix.builds.features }}" == "" ]]; then @@ -155,17 +132,6 @@ jobs: TARGET_BINS+="--bin ${BIN_FILE} " done echo "TARGET_BINS=${TARGET_BINS}" >> $GITHUB_ENV - TARGET_LIBS="" - if [[ "${{ matrix.builds.target_libs }}" == "" ]]; then - ARRAY_LIBS=( $(echo ${TS_LIBRARIES} | tr ', ' '\n') ) - else - ARRAY_LIBS=( $(echo "${{ matrix.builds.target_libs }}" | tr ', ' '\n') ) - fi - for LIB_FILE in "${ARRAY_LIBS[@]}"; do - echo "Adding ${LIB_FILE} to library Builds" - TARGET_LIBS+="--package ${LIB_FILE} " - done - echo "TARGET_LIBS=${TARGET_LIBS}" >> $GITHUB_ENV TARI_BUILD_ISA_CPU=${{ matrix.builds.target }} # Strip unknown part TARI_BUILD_ISA_CPU=${TARI_BUILD_ISA_CPU//-unknown-linux-gnu} @@ -203,8 +169,9 @@ jobs: - name: Install macOS dependencies if: startsWith(runner.os,'macOS') run: | - # openssl, cmake and autoconf already installed - brew install zip coreutils automake protobuf + # Already installed items + # brew install openssl cmake autoconf zip + brew install coreutils automake protobuf rustup target add ${{ matrix.builds.target }} - name: Install Windows dependencies @@ -312,14 +279,6 @@ jobs: ${{ env.TARGET_BINS }} \ ${{ matrix.builds.flags }} --locked - - name: Build release libraries - shell: bash - run: | - ${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \ - --target ${{ matrix.builds.target }} \ - --lib ${{ env.TARGET_LIBS }} \ - ${{ matrix.builds.flags }} --locked - - name: Copy binaries to folder for archiving shell: bash run: | @@ -354,41 +313,8 @@ jobs: cp -vf "${GITHUB_WORKSPACE}/target/${{ matrix.builds.target }}/release/${FILE}${LIB_EXT}" . fi done - if [ -f "${GITHUB_WORKSPACE}/applications/minotari_node/${PLATFORM_SPECIFIC_DIR}/runtime/start_tor${SHELL_EXT}" ]; then - cp -vf "${GITHUB_WORKSPACE}/applications/minotari_node/${PLATFORM_SPECIFIC_DIR}/runtime/start_tor${SHELL_EXT}" . - fi ls -alhtR ${{ env.MTS_SOURCE }} - - name: Build minotari_node with metrics too - if: ${{ matrix.builds.build_metric }} - shell: bash - run: | - ${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \ - --target ${{ matrix.builds.target }} \ - --features "${{ env.BUILD_FEATURES }}, metrics" \ - --bin minotari_node \ - ${{ matrix.builds.flags }} --locked - cp -vf "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/minotari_node${TS_EXT}" \ - "${{ env.MTS_SOURCE }}/minotari_node-metrics${TS_EXT}" - - - name: Build targeted miners - # if: ${{ ( startsWith(github.ref, 'refs/tags/v') ) && ( matrix.builds.miner_cpu_targets != '' ) }} - if: ${{ matrix.builds.miner_cpu_targets != '' }} - shell: bash - run: | - ARRAY_TARGETS=( $(echo "${{ matrix.builds.miner_cpu_targets }}" | tr ', ' '\n') ) - for CPU_TARGET in "${ARRAY_TARGETS[@]}"; do - echo "Target CPU ${CPU_TARGET} for miner" - export RUSTFLAGS="-C target-cpu=${CPU_TARGET}" - ${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \ - --target ${{ matrix.builds.target }} \ - --features "${{ env.BUILD_FEATURES }}" \ - --bin minotari_miner \ - ${{ matrix.builds.flags }} --locked - cp -vf "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/minotari_miner" \ - "${{ env.MTS_SOURCE }}/minotari_miner-${CPU_TARGET}" - done - - name: Pre/unsigned OSX Artifact upload for Archive # Debug if: ${{ false }} @@ -505,6 +431,7 @@ jobs: - name: Build the Windows installer if: startsWith(runner.os,'Windows') + continue-on-error: true shell: cmd run: | cd buildtools @@ -519,6 +446,7 @@ jobs: - name: Artifact upload for Windows installer if: startsWith(runner.os,'Windows') + continue-on-error: true uses: actions/upload-artifact@v4 with: name: "${{ env.TS_FILENAME }}_windows_installer" @@ -549,36 +477,6 @@ jobs: name: ${{ env.TS_FILENAME }}_archive-${{ matrix.builds.name }} path: "${{ github.workspace }}${{ env.TS_DIST }}/${{ env.BINFILE }}.zip*" - - name: Prep diag-utils archive for upload - continue-on-error: true - shell: bash - run: | - mkdir -p "${{ env.MTS_SOURCE }}-diag-utils" - cd "${{ env.MTS_SOURCE }}-diag-utils" - # Find RandomX built tools for testing - find "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/" \ - -name "randomx-*${{ env.TS_EXT}}" -type f -perm -+x -exec cp -vf {} . \; - echo "Compute diag utils shasum" - ${SHARUN} * \ - >> "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256" - cat "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256" - echo "Checksum verification for diag utils is " - ${SHARUN} --check "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256" - 7z a "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip" * - echo "Compute diag utils archive shasum" - ${SHARUN} "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip" \ - >> "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256" - cat "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256" - echo "Checksum verification for diag utils archive is " - ${SHARUN} --check "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256" - - - name: Artifact upload for diag-utils - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: ${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }} - path: "${{ github.workspace }}${{ env.TS_DIST }}-diag-utils/*.zip*" - macOS-universal-assemble: name: macOS universal assemble needs: builds @@ -779,7 +677,6 @@ jobs: needs: builds env: - TARI_NETWORK_DIR: ${{ needs.builds.outputs.TARI_NETWORK_DIR }} TARI_VERSION: ${{ needs.builds.outputs.TARI_VERSION }} permissions: @@ -837,10 +734,6 @@ jobs: ls -alhtR echo "Clean up" # Bash check if file with wildcards, does not work as expected - # if [ -f ${{ env.TS_FILENAME }}*diag-utils* ] ; then - if ls ${{ env.TS_FILENAME }}*diag-utils* > /dev/null 2>&1 ; then - rm -fv ${{ env.TS_FILENAME }}*diag-utils* - fi echo "Folder setup" if ls ${{ env.TS_FILENAME }}*linux* > /dev/null 2>&1 ; then mkdir -p "linux/${{ env.TARI_NETWORK_DIR }}/" diff --git a/.license.ignore b/.license.ignore new file mode 100644 index 00000000..e69de29b diff --git a/scripts/cross_compile_tooling.sh b/scripts/cross_compile_tooling.sh new file mode 100755 index 00000000..625c3403 --- /dev/null +++ b/scripts/cross_compile_tooling.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Move all cross-compiling steps into a single sourced script +# + +set -e + +printenv + +if [ "${TARGETARCH}" = "arm64" ] ; then + platform_env=aarch64 + export BUILD_TARGET="${platform_env}-unknown-linux-gnu/" + export RUST_TARGET="--target=${platform_env}-unknown-linux-gnu" + #export ARCH=${ARCH:-generic} + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=${platform_env}-linux-gnu-gcc + export CC_aarch64_unknown_linux_gnu=${platform_env}-linux-gnu-gcc + export CXX_aarch64_unknown_linux_gnu=${platform_env}-linux-gnu-g++ + export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/${platform_env}-linux-gnu/include/" + #export RUSTFLAGS="-C target_cpu=$ARCH" + #export ROARING_ARCH=$ARCH + rustup target add ${platform_env}-unknown-linux-gnu + rustup toolchain install stable-${platform_env}-unknown-linux-gnu --force-non-host + + # Check for Debian + if [ -f "/etc/debian_version" ] ; then + dpkg --add-architecture ${TARGETARCH} + apt-get update || true + apt-get install -y pkg-config libssl-dev:${TARGETARCH} gcc-${platform_env}-linux-gnu g++-${platform_env}-linux-gnu + export AARCH64_UNKNOWN_LINUX_GNU_OPENSSL_INCLUDE_DIR=/usr/include/${platform_env}-linux-gnu/openssl/ + export PKG_CONFIG_ALLOW_CROSS=1 + fi + +elif [ "${TARGETARCH}" = "amd64" ] ; then + platform_env=x86_64 + platform_env_alt=x86-64 + export BUILD_TARGET="${platform_env}-unknown-linux-gnu/" + export RUST_TARGET="--target=${platform_env}-unknown-linux-gnu" + # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html + #export ARCH=${ARCH:-x86-64} + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=${platform_env}-linux-gnu-gcc + export CC_x86_64_unknown_linux_gnu=${platform_env_alt}-linux-gnu-gcc + export CXX_x86_64_unknown_linux_gnu=${platform_env_alt}-linux-gnu-g++ + export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/${platform_env}-linux-gnu/include/" + rustup target add ${platform_env}-unknown-linux-gnu + rustup toolchain install stable-${platform_env}-unknown-linux-gnu --force-non-host + + # Check for Debian + if [ -f "/etc/debian_version" ] ; then + dpkg --add-architecture ${TARGETARCH} + apt-get update + apt-get install -y pkg-config libssl-dev:${TARGETARCH} gcc-${platform_env_alt}-linux-gnu g++-${platform_env_alt}-linux-gnu + export X86_64_UNKNOWN_LINUX_GNU_OPENSSL_INCLUDE_DIR=/usr/include/${platform_env}-linux-gnu/openssl/ + export PKG_CONFIG_ALLOW_CROSS=1 + fi + +else + echo "Need to source [ ${0##*/} ] with env TARGETARCH set to either arm64 or amd64" + exit 1 +fi diff --git a/scripts/cross_compile_ubuntu_18-pre-build.sh b/scripts/cross_compile_ubuntu_18-pre-build.sh new file mode 100755 index 00000000..e93f2a10 --- /dev/null +++ b/scripts/cross_compile_ubuntu_18-pre-build.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# +# Single script for Ubuntu 18.04 package setup, mostly used for cross-compiling +# + +set -e + +USAGE="Usage: $0 target build ie: x86_64-unknown-linux-gnu or aarch64-unknown-linux-gnu" + +if [ "$#" == "0" ]; then + echo "$USAGE" + exit 1 +fi + +if [ -z "${CROSS_DEB_ARCH}" ]; then + echo "Should be run from cross, which sets the env CROSS_DEB_ARCH" + exit 1 +fi + +targetBuild="${1}" +nativeRunTime=$(uname -m) +echo "Native RunTime is ${nativeRunTime}" + +if [ "${nativeRunTime}" == "x86_64" ]; then + nativeArch=amd64 + if [ "${targetBuild}" == "aarch64-unknown-linux-gnu" ]; then + targetArch=arm64 + targetPlatform=aarch64 + else + targetArch=amd64 + targetPlatform=x86-64 + fi +elif [ "${nativeRunTime}" == "aarch64" ]; then + nativeArch=arm64 + if [ "${targetBuild}" == "x86_64-unknown-linux-gnu" ]; then + targetArch=amd64 + targetPlatform=x86-64 + fi +elif [ "${nativeRunTime}" == "riscv64" ]; then + nativeArch=riscv64 + echo "ToDo!" +else + echo "!!Unsupport platform!!" + exit 1 +fi + +crossArch=${CROSS_DEB_ARCH} +apt-get update + +# Base install packages +# scripts/install_ubuntu_dependencies.sh +apt-get install --no-install-recommends --assume-yes \ + apt-transport-https \ + ca-certificates \ + curl \ + gpg \ + bash \ + less \ + openssl \ + libssl-dev \ + pkg-config \ + libsqlite3-dev \ + libsqlite3-0 \ + libreadline-dev \ + git \ + cmake \ + dh-autoreconf \ + clang \ + g++ \ + libc++-dev \ + libc++abi-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libncurses5-dev \ + libncursesw5-dev \ + libudev-dev \ + libhidapi-dev \ + zip + +echo "Installing rust ..." +mkdir -p "$HOME/.cargo/bin/" +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +export PATH="$HOME/.cargo/bin:$PATH" +. "$HOME/.cargo/env" + +# Cross-CPU compile setup +if [ "${CROSS_DEB_ARCH}" != "${nativeArch}" ]; then + echo "Setup Cross CPU Compile ..." + sed -i.save -e "s/^deb\ http/deb [arch="${nativeArch}"] http/g" /etc/apt/sources.list + + . /etc/lsb-release + ubuntu_tag=${DISTRIB_CODENAME} + + if [ "${crossArch}" == "arm64" ]; then + cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse + +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-updates main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-updates main restricted universe multiverse + +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-backports main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-backports main restricted universe multiverse + +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-security main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-security main restricted universe multiverse + +deb [arch=${crossArch}] http://archive.canonical.com/ubuntu ${ubuntu_tag} partner +# deb-src [arch=${crossArch}] http://archive.canonical.com/ubuntu ${ubuntu_tag} partner +EoF + fi + + if [ "${crossArch}" == "amd64" ]; then + cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} main restricted +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} main restricted + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates main restricted +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates main restricted + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} universe +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} universe +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates universe +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates universe + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} multiverse +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} multiverse +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates multiverse +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates multiverse + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-backports main restricted universe multiverse +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-backports main restricted universe multiverse + +# deb http://archive.canonical.com/ubuntu ${ubuntu_tag} partner +# deb-src http://archive.canonical.com/ubuntu ${ubuntu_tag} partner + +deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security main restricted +# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security main restricted +deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security universe +# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security universe +deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multiverse +# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multiverse +EoF + fi + + dpkg --add-architecture ${CROSS_DEB_ARCH} + apt-get update + + # scripts/install_ubuntu_dependencies-cross_compile.sh x86-64 + apt-get --assume-yes install \ + pkg-config-${targetPlatform}-linux-gnu \ + gcc-${targetPlatform}-linux-gnu \ + g++-${targetPlatform}-linux-gnu + + # packages needed for Ledger and hidapi + apt-get --assume-yes install \ + libudev-dev:${CROSS_DEB_ARCH} \ + libhidapi-dev:${CROSS_DEB_ARCH} + +fi + +rustup target add ${targetBuild} +rustup toolchain install stable-${targetBuild} --force-non-host + +rustup target list --installed +rustup toolchain list diff --git a/scripts/file_license_check.sh b/scripts/file_license_check.sh new file mode 100755 index 00000000..8985052d --- /dev/null +++ b/scripts/file_license_check.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Must be run from the repo root +# + +set -e + +# rg -i "Copyright.*The Tari Project" --files-without-match \ +# -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue}' . \ +# | sort > /tmp/rgtemp + +# Exclude files without extensions as well as those with extensions that are not in the list +# +rgTemp=$(mktemp) +rg -i "Copyright.*The Tari Project" --files-without-match \ + -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue}' . \ + | while IFS= read -r file; do + if [[ -n $(basename "$file" | grep -E '\.') ]]; then + echo "$file" + fi + done | sort > ${rgTemp} + +# Sort the .license.ignore file as sorting seems to behave differently on different platforms +licenseIgnoreTemp=$(mktemp) +cat .license.ignore | sort > ${licenseIgnoreTemp} + +DIFFS=$(diff -u --strip-trailing-cr ${licenseIgnoreTemp} ${rgTemp}) + +# clean up +rm -vf ${rgTemp} +rm -vf ${licenseIgnoreTemp} + +if [ -n "$DIFFS" ]; then + echo "New files detected that either need copyright/license identifiers added, or they need to be added to .license.ignore" + echo "NB: The ignore file must be sorted alphabetically!" + + echo "Diff:" + echo "$DIFFS" + exit 1 +fi diff --git a/scripts/install_ubuntu_dependencies-cross_compile.sh b/scripts/install_ubuntu_dependencies-cross_compile.sh new file mode 100755 index 00000000..79ce3310 --- /dev/null +++ b/scripts/install_ubuntu_dependencies-cross_compile.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +# +# Install Ubuntu aarch64(arm64)/riscv64 deb dev/tool packages on x86_64 +# + +set -e + +USAGE="Usage: $0 ISA_ARCH other packages, ie aarch64" + +if [ "$#" == "0" ]; then + echo "$USAGE" + exit 1 +fi + +isa_arch=${1} +shift + +apt-get --assume-yes install $* \ + pkg-config-${isa_arch}-linux-gnu \ + gcc-${isa_arch}-linux-gnu \ + g++-${isa_arch}-linux-gnu diff --git a/scripts/install_ubuntu_dependencies-rust-arm64.sh b/scripts/install_ubuntu_dependencies-rust-arm64.sh new file mode 100755 index 00000000..a17edcae --- /dev/null +++ b/scripts/install_ubuntu_dependencies-rust-arm64.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +# +# Install rust target/toolchain for aarch64/arm64 +# + +set -e + +export PATH="$HOME/.cargo/bin:$PATH" +rustup target add aarch64-unknown-linux-gnu +rustup toolchain install stable-aarch64-unknown-linux-gnu --force-non-host diff --git a/scripts/install_ubuntu_dependencies-rust.sh b/scripts/install_ubuntu_dependencies-rust.sh new file mode 100755 index 00000000..3ac1a9f1 --- /dev/null +++ b/scripts/install_ubuntu_dependencies-rust.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +# +# Install rust unattended - https://www.rust-lang.org/tools/install +# + +set -e + +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +export PATH="$HOME/.cargo/bin:$PATH" +. "$HOME/.cargo/env" diff --git a/scripts/install_ubuntu_dependencies.sh b/scripts/install_ubuntu_dependencies.sh new file mode 100755 index 00000000..b326d220 --- /dev/null +++ b/scripts/install_ubuntu_dependencies.sh @@ -0,0 +1,27 @@ +apt-get install --no-install-recommends --assume-yes \ + apt-transport-https \ + ca-certificates \ + curl \ + gpg \ + bash \ + less \ + openssl \ + libssl-dev \ + pkg-config \ + libsqlite3-dev \ + libsqlite3-0 \ + libreadline-dev \ + git \ + cmake \ + dh-autoreconf \ + clang \ + g++ \ + libc++-dev \ + libc++abi-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libncurses5-dev \ + libncursesw5-dev \ + libudev-dev \ + libhidapi-dev \ + zip