ci.yaml

name: test
on:
  push:
    branches:
      - "main"
      - "release-*"
  pull_request:
    branches: [main]
jobs:
  ui:
    name: UI
    runs-on: ubuntu-latest
    timeout-minutes: 6
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: "16"
      - name: Setup Node-Cache
        uses: actions/cache@v3
        with:
          path: ui/node_modules
          key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }}
      # Temporarily disable it.
      #- name: Yarn Build/Test
      #  run: make ui-test
      - name: Ensure nothing changed
        run: git diff --exit-code
  codegen:
    name: Codegen
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      - name: Restore go build cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v1-${{ hashFiles('**/go.mod') }}
      - name: Setup Golang
        uses: actions/setup-go@v4.0.0
        with:
          go-version: '1.20'
      - name: Add bins to PATH
        run: |
          echo /home/runner/go/bin >> $GITHUB_PATH
          echo /usr/local/bin >> $GITHUB_PATH
      - name: Install protoc
        run: |
          set -eux -o pipefail
          PROTOC_VERSION=3.19.4
          PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip
          curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP
          sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
          sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*'
          sudo chmod +x /usr/local/bin/protoc
          sudo find /usr/local/include -type f | xargs sudo chmod a+r
          sudo find /usr/local/include -type d | xargs sudo chmod a+rx
          rm -f $PROTOC_ZIP
          ls /usr/local/include/google/protobuf/
      - name: Install pandoc
        run: |
          set -eux -o pipefail
          PANDOC_VERSION=2.17.1
          PANDOC_ZIP=pandoc-$PANDOC_VERSION-linux-amd64.tar.gz
          curl -OL https://github.com/jgm/pandoc/releases/download/$PANDOC_VERSION/$PANDOC_ZIP
          sudo tar xvzf $PANDOC_ZIP --strip-components 1 -C /usr/local
          rm -f $PANDOC_ZIP
          echo /usr/local/pandoc-$PANDOC_VERSION/bin >> $GITHUB_PATH
      - name: Get dependencies
        run: go mod download
      - name: Make codegen
        run: |
          echo 'GOPATH=/home/runner/go' >> $GITHUB_ENV
          make -B codegen
      - name: Ensure nothing changed
        run: git diff --exit-code

  unit-tests:
    name: Unit Tests
    runs-on: ubuntu-latest
    timeout-minutes: 10
    services:
      redis:
        image: redis:6.2
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - name: Set up Go 1.x
        uses: actions/setup-go@v4.0.0
        with:
          go-version: '1.20'
        id: go
      - name: Check out code into the Go module directory
        uses: actions/checkout@v3
      - name: Restore Go build cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
      - name: Get dependencies
        run: go mod download
      - name: Test
        run: make test-code

  lint:
    name: Lint
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      GOPATH: /home/runner/go
    steps:
      - uses: actions/checkout@v3
      - name: Setup Golang
        uses: actions/setup-go@v4.0.0
        with:
          go-version: '1.20'
      - name: Restore Go build cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
      - run: make lint
      - run: git diff --exit-code

  e2e-tests:
    name: E2E Tests
    runs-on: ubuntu-latest
    timeout-minutes: 20
    strategy:
      fail-fast: false
      max-parallel: 12
      matrix:
        driver: [jetstream]
        case: [e2e, diamond-e2e, transformer-e2e, kafka-e2e, http-e2e, nats-e2e, sdks-e2e, reduce-e2e, udsource-e2e, api-e2e, sideinputs-e2e, idle-source-e2e]
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      - name: Restore go build cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v2-${{ hashFiles('**/go.mod') }}
      - name: Setup Golang
        uses: actions/setup-go@v4.0.0
        with:
          go-version: '1.20'
      - name: Add bins to PATH
        run: |
          echo /home/runner/go/bin >> $GITHUB_PATH
          echo /usr/local/bin >> $GITHUB_PATH
      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: "16"
      - name: Setup Node-Cache
        uses: actions/cache@v3
        with:
          path: ui/node_modules
          key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }}
      - name: Install k3d
        run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash &
      - name: Create a registry and a cluster
        run: |
          k3d registry create e2e-registry --port 5111
          k3d cluster create e2e --registry-use k3d-e2e-registry:5111
          k3d kubeconfig get e2e > ~/.kube/numaflow-e2e-config
          echo '127.0.0.1 k3d-e2e-registry' | sudo tee -a /etc/hosts
      - name: Install Numaflow
        env:
          GOPATH: /home/runner/go
        run: |
          KUBECONFIG=~/.kube/numaflow-e2e-config IMAGE_NAMESPACE=k3d-e2e-registry:5111 VERSION=${{ github.sha }} DOCKER_PUSH=true make start
      - name: Run tests
        env:
          GOPATH: /home/runner/go
        run: KUBECONFIG=~/.kube/numaflow-e2e-config IMAGE_NAMESPACE=k3d-e2e-registry:5111 VERSION=${{ github.sha }} DOCKER_PUSH=true ISBSVC=${{matrix.driver}} make test-${{matrix.case}}