azure-pipelines.yml

# Azure Pipelines CI build configuration
# Documentation at https://aka.ms/yaml

variables:
  VERILATOR_VERSION: 4.010
  VERILATOR_PATH: /opt/buildcache/verilator/$(VERILATOR_VERSION)
  TOOLCHAIN_PATH: /opt/buildcache/riscv
  # Release tag from https://github.com/lowRISC/lowrisc-toolchains/releases
  TOOLCHAIN_VERSION: 20191010-1
  # This controls where builds happen, and gets picked up by build_consts.sh.
  BUILD_ROOT: $(Build.ArtifactStagingDirectory)

trigger:
  tags:
    include:
    - '*'
  # Combine builds on master as long as another build is running
  batch: true
  branches:
    include:
    - master

jobs:
- job: "lint"
  displayName: "Run code quality checks (lint)"
  pool:
    vmImage: "ubuntu-16.04"
  steps:
  - bash: |
      sudo apt-get remove -y clang-6.0 libclang-common-6.0-dev libclang1-6.0 libllvm6.0
      sudo apt-get install -y python3 python3-pip build-essential srecord python3-setuptools zlib1g-dev libusb-1.0 clang-format
      sudo pip3 install -U -r python-requirements.txt
    displayName: 'Install dependencies'

  - bash: |
      python3 --version
      yapf --version
      isort --version
      clang-format -version
    displayName: 'Display tool versions'

#   XXX: Python lint checks are disabled until Issue #313 is resolved
#    - bash: find ./util -iname '*.py' -print0 | xargs -0 -n1 $PWD/util/lintpy.py -f
#      displayName: 'Run Python lint'

  - bash: |
      make -C hw regs && git diff --exit-code
      if [ $? != 0 ]; then
        echo -n "##vso[task.logissue type=error]"
        echo "Register headers not up-to-date. Regenerate them with 'make -C hw regs'."
        exit 1
      fi
    condition: always()
    displayName: 'Ensure all generated files are clean and up-to-date'

  - bash: |
      ./util/build_docs.py
    condition: always()
    displayName: 'Render documentation'

  - bash: |
      cd site/landing
      ../../build/docs-hugo/hugo
    condition: always()
    displayName: 'Render landing site'

  - bash: |
      # XXX: As of today, task.logissue comments with 'sourcepath' set don't
      # get reported to GitHub Checks annotations. Upstream bug report:
      # https://developercommunity.visualstudio.com/content/problem/689794/pipelines-logging-command-logissue-does-not-report.html
      #echo "##vso[task.issue type=error;sourcepath=/azure-pipelines.yml;linenumber=45;columnnumber=1;code=100;]Found something that could be a problem."
      fork_origin=$(git merge-base --fork-point origin/master)
      changed_files=$(git diff --name-only $fork_origin | grep -v /vendor/ | grep -E '\.(cpp|cc|c|h)$')
      test -z "$changed_files" || git diff -U0 $fork_origin $changed_files | clang-format-diff -p1 | tee clang-format-output
      if [ -s clang-format-output ]; then
        echo -n "##vso[task.logissue type=error]"
        echo "C/C++ lint failed. Use 'git clang-format' with appropriate options to reformat the changed code."
        exit 1
      fi
    # This check is not idempotent, but checks changes to a base branch.
    # Run it only on pull requests.
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: 'Use clang-format to check C/C++ coding style'

  - bash: |
      commit_range=$(git merge-base --fork-point origin/master)..HEAD
      # Notes:
      # * Merge commits are not checked. We always use rebases instead of
      #   merges to keep a linear history, which makes merge commits disappear
      #   ultimately, making them only a CI artifact which should not be
      #   checked.
      # * 'type=error' is used even for warnings. Only "errors" are shown in
      #   the GitHub checks API. However, warnings don't return a non-zero
      #   error code and don't fail the build step.
      ./util/lint_commits.py \
          --no-merges \
          --error-msg-prefix='##vso[task.logissue type=error]' \
          --warning-msg-prefix='##vso[task.logissue type=error]' \
          $commit_range
    # Only run on pull requests to check new commits only
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: "Check commit metadata"

  - bash: |
      only_doc_changes=0
      if [ "$(Build.Reason)" = "PullRequest" ]; then
        # Conservative way of checking for documentation-only changes.
        # Only relevant for pipelines triggered from pull requests
        echo "Checking for doc-only changes in this pull request"
        fork_origin=$(git merge-base --fork-point origin/master)
        only_doc_changes=$(git diff --name-only $fork_origin | grep -v '\.md$' -q; echo $?)
      fi
      echo "##vso[task.setvariable variable=onlyDocChanges;isOutput=true]${only_doc_changes}"
    displayName: "Check if the commit only contains documentation changes"
    name: DetermineBuildType

- job: "sw_build"
  displayName: "Build Software"
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool:
    vmImage: "ubuntu-16.04"
  steps:
  - bash: |
      sudo apt-get install -y python3 python3-pip build-essential pkgconf srecord python3-setuptools zlib1g-dev libusb-1.0 libftdi1-dev libftdi1-2 libssl-dev ninja-build \
        && sudo pip3 install -U -r python-requirements.txt
    displayName: 'Install dependencies'
  - bash: |
      set -x
      sudo util/get-toolchain.py \
        --target-dir="${TOOLCHAIN_PATH}" \
        --release-version="${TOOLCHAIN_VERSION}" \
        --update \
        --force
    displayName: 'Install toolchain'
  - bash: |
      . util/build_consts.sh
      ./meson_init.sh -f
      ninja -C "$(sw_obj_dir sim-verilator)" all
      ninja -C "$(sw_obj_dir fpga)" all
    displayName: 'Build embedded targets'
  - template: 'ci/upload-artifacts-template.yml'
    parameters:
      artifact: sw_build

- job: "deprecated_make_build"
  displayName: "Build Software with Make (deprecated)"
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool: Default
  steps:
  - bash: |
      sudo apt-get install -y python3 python3-pip build-essential srecord python3-setuptools zlib1g-dev libusb-1.0 libftdi1-dev libftdi1-2 libssl-dev \
        && sudo pip3 install -U -r python-requirements.txt
    displayName: 'Install dependencies'
  - bash: |
      export TOOLCHAIN_PATH="${TOOLCHAIN_PATH}"
      export TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}"
      export DIST_DIR="$(Build.ArtifactStagingDirectory)/dist"
      REQUEST_UPDATE=true ci/run_sw_make.sh
    displayName: 'Build embedded targets'
  - bash: |
      make -C sw/host/spiflash clean all
      # TODO: build system updates needed to copy build output to a dist
      # staging directory.
      mkdir -p $(Build.ArtifactStagingDirectory)/dist/sw/host/spiflash
      cp sw/host/spiflash/spiflash $(Build.ArtifactStagingDirectory)/dist/sw/host/spiflash
    displayName: 'Build host targets'

- job: "top_earlgrey_verilator"
  displayName: "Build Verilator simulation of the Earl Grey toplevel design"
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool: Default
  steps:
  - bash: |
      sudo apt-get install -y python3 python3-pip build-essential srecord python3-setuptools zlib1g-dev libusb-1.0 libelf-dev \
        && sudo pip3 install -U -r python-requirements.txt \
        && sudo apt-get install git make autoconf g++ flex bison curl
    displayName: 'Install dependencies'
  - bash: |
      set -e
      if [ ! -d $(VERILATOR_PATH) ]; then
        echo "Building verilator (no cached build found)"
        mkdir -p build/verilator
        cd build/verilator
        curl -Ls -o verilator.tgz https://www.veripool.org/ftp/verilator-$(VERILATOR_VERSION).tgz
        tar -xf verilator.tgz
        cd verilator-$(VERILATOR_VERSION)
        ./configure --prefix=$(VERILATOR_PATH)
        make -j$(nproc)
        mkdir -p $VERILATOR_PATH
        make install
      else
        echo "Re-using cached verilator build"
      fi
    displayName: 'Build and install Verilator'
  - bash: |
      export PATH=$VERILATOR_PATH/bin:$PATH
      python3 --version
      fusesoc --version
      verilator --version
    displayName: 'Display environment'
  - bash: |
      . util/build_consts.sh
      mkdir -p "$OBJ_DIR/hw"
      mkdir -p "$BIN_DIR/hw/top_earlgrey"

      export PATH=$VERILATOR_PATH/bin:$PATH
      fusesoc --cores-root=. \
        run --target=sim --setup --build \
        --build-root="$OBJ_DIR/hw" \
        lowrisc:systems:top_earlgrey_verilator

      cp "$OBJ_DIR/hw/sim-verilator/Vtop_earlgrey_verilator" \
        "$BIN_DIR/hw/top_earlgrey"
    displayName: 'Build simulation with Verilator'
  - template: 'ci/upload-artifacts-template.yml'
    parameters:
      artifact: top_earlgrey_verilator

- job: "top_earlgrey_nexysvideo"
  displayName: "Build NexysVideo variant of the Earl Grey toplevel design using Vivado"
  dependsOn:
    - lint
    # The bootrom is built into the FPGA image at synthesis time.
    - sw_build
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool: Default
  timeoutInMinutes: 120 # 2 hours
  steps:
  - bash: |
      sudo apt-get install -y python3 python3-pip build-essential srecord python3-setuptools zlib1g-dev libusb-1.0 tree \
        && sudo pip3 install -U -r python-requirements.txt
    displayName: 'Install dependencies'
  - template: 'ci/download-artifacts-template.yml'
  - bash: |
      set -e
      . util/build_consts.sh
      mkdir -p "$OBJ_DIR/hw"
      mkdir -p "$BIN_DIR/hw/top_earlgrey"

      BOOTROM_VMEM="$BIN_DIR/sw/device/fpga/boot_rom/boot_rom.vmem"
      test -f "$BOOTROM_VMEM"

      . /opt/xilinx/Vivado/2018.3/settings64.sh
      fusesoc --cores-root=. \
        run --target=synth --setup --build \
        --build-root="$OBJ_DIR/hw" \
        lowrisc:systems:top_earlgrey_nexysvideo \
        --ROM_INIT_FILE="$BOOTROM_VMEM"

      cp "$OBJ_DIR/hw/synth-vivado/lowrisc_systems_top_earlgrey_nexysvideo_0.1.bit" \
        "$BIN_DIR/hw/top_earlgrey"
    displayName: 'Build bitstream with Vivado'
  - template: 'ci/upload-artifacts-template.yml'
    parameters:
      artifact: top_earlgrey_nexysvideo

- job: "execute_verilated_tests"
  displayName: "Execute tests on the Verilated system"
  pool: "Default"
  dependsOn:
    - top_earlgrey_verilator
    - sw_build
  steps:
  - bash: |
      sudo apt-get install -y python3 python3-pip build-essential python3-setuptools libelf1 tree
      sudo pip3 install -r python-requirements.txt
    displayName: 'Install dependencies'
  - template: 'ci/download-artifacts-template.yml'
  - bash: |
      . util/build_consts.sh
      export VERILATED_SYSTEM_PATH="$BIN_DIR/hw/top_earlgrey/Vtop_earlgrey_verilator"
      ci/run_verilator_pytest.sh
    displayName: 'Execute tests'

- job: "deploy_releaseartifacts"
  displayName: "Package and deploy release distribution"
  pool:
    vmImage: "ubuntu-latest"
  dependsOn:
    - lint
    - sw_build
    - top_earlgrey_verilator
    - top_earlgrey_nexysvideo
  condition: eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0')
  steps:
  - bash: |
      sudo apt-get install -y tree
    displayName: 'Install dependencies'
  - template: 'ci/download-artifacts-template.yml'
  - bash: |
      . util/build_consts.sh

      util/make_distribution.sh

      tar --list -f $BIN_DIR/opentitan-*.tar.xz
      # Put the resulting tar file into a directory the |publish| step below can reference.
      mkdir "$BUILD_ROOT/dist-final"
      mv $BIN_DIR/opentitan-*.tar.xz "$BUILD_ROOT/dist-final"
    displayName: 'Create final dist directory out of partial ones'
  - publish: $(Build.ArtifactStagingDirectory)/dist-final
    artifact: opentitan-dist
    displayName: "Upload release artifacts as Azure artifact"
  - task: GithubRelease@0
    displayName: 'Upload to GitHub releases (only tags)'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
    inputs:
      gitHubConnection: opentitan-release-upload
      repositoryName: lowrisc/opentitan
      addChangeLog: false
      assets: |
          $(Build.ArtifactStagingDirectory)/dist-final/*