deploy-stage.yaml

name: Deploy connector to dockerhub, release cli on github
on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
    tags:
      - 'v*'
  workflow_dispatch:

# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  build-and-push-image:
    runs-on: ubuntu-latest
    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
      - name: Build and push Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: ${{ startsWith(github.ref, 'refs/tags/v') }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}

  build-cli-binaries:
    name: build the CLI binaries
    strategy:
      matrix:
        include:
          - runner: ubuntu-latest
            target: x86_64-unknown-linux-musl
            rustflags: -C target-feature=+crt-static
            linux-packages: musl-tools
          - runner: ubuntu-latest
            target: aarch64-unknown-linux-musl
            rustflags: -C target-feature=+crt-static
            linux-packages: gcc-aarch64-linux-gnu musl-tools
            linker: /usr/bin/aarch64-linux-gnu-gcc
          - runner: macos-latest
            target: x86_64-apple-darwin
          - runner: macos-latest
            target: aarch64-apple-darwin
          - runner: windows-latest
            target: x86_64-pc-windows-msvc
            rustflags: -C target-feature=+crt-static
            extension: .exe
    runs-on: ${{ matrix.runner }}
    env:
      CARGO_BUILD_TARGET: ${{ matrix.target }}
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUSTFLAGS: "-D warnings ${{ matrix.rustflags }}"
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v4

      - name: install protoc
        uses: arduino/setup-protoc@v3
        with:
          version: "25.x"
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: install tools
        run: |
          rustup show
          rustup target add ${{ matrix.target }}

      - name: install other packages required
        if: matrix.linux-packages
        run: |
          sudo apt-get update
          sudo apt-get install -y ${{ matrix.linux-packages }}

      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: "build" # share the cache across jobs

      - name: build the CLI
        run: |
          # If we're on a tag, use the tag name as the release version.
          if [[ "$GITHUB_REF_TYPE" == 'tag' ]]; then
            # Ensure that the version specified in Cargo.toml is the same as the tag (with a 'v' prefix).
            CARGO_VERSION="$(cargo metadata --format-version=1 | jq -r '.packages | .[] | select(.name == "ndc-clickhouse-cli") | .version')"
            echo "Git tag: ${GITHUB_REF_NAME}"
            echo "Cargo version: ${CARGO_VERSION}"

            if [[ "${GITHUB_REF_NAME}" != "v${CARGO_VERSION}" ]]; then
              echo >&2 "The Git tag is \"${GITHUB_REF_NAME}\", but the version in Cargo.toml is \"${CARGO_VERSION}\"."
              echo >&2 'These must be the same, with a "v" prefix for the tag. Aborting.'
              exit 1
            fi
            export RELEASE_VERSION="$GITHUB_REF_NAME"
            echo "RELEASE_VERSION = ${RELEASE_VERSION}"
          fi

          if [[ -n '${{ matrix.linker }}' ]]; then
            TARGET_SCREAMING="$(echo '${{ matrix.target }}' | tr '[:lower:]' '[:upper:]' | tr '-' '_')"
            echo "CARGO_TARGET_${TARGET_SCREAMING}_LINKER"='${{ matrix.linker }}'
            declare "CARGO_TARGET_${TARGET_SCREAMING}_LINKER"='${{ matrix.linker }}'
            export "CARGO_TARGET_${TARGET_SCREAMING}_LINKER"
          fi

          echo "Building for target: ${CARGO_BUILD_TARGET}"
          cargo build --locked --release --package ndc-clickhouse-cli

          mkdir -p release
          mv -v target/${{ matrix.target }}/release/ndc-clickhouse-cli release/ndc-clickhouse-cli-${{ matrix.target }}${{ matrix.extension }}

      - uses: actions/upload-artifact@v4
        with:
          name: ndc-clickhouse-cli-${{ matrix.target }}${{ matrix.extension }}
          path: release
          if-no-files-found: error

  release:
    name: release to GitHub
    needs:
      - build-and-push-image
      - build-cli-binaries
    runs-on: ubuntu-latest
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    steps:
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          rustflags: "" # defaults to "-D warnings", set to empty string to allow warnings

      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          path: release/artifacts
          merge-multiple: true

      - name: generate CLI manifest
        run: |
          set -evo pipefail
          ROOT="$(pwd)"

          export CLI_VERSION="$GITHUB_REF_NAME"

          export LINUX_AMD64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-x86_64-unknown-linux-musl     | cut -f1 -d' ')
          export MACOS_AMD64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-x86_64-apple-darwin           | cut -f1 -d' ')
          export WINDOWS_AMD64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-x86_64-pc-windows-msvc.exe  | cut -f1 -d' ')
          export LINUX_ARM64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-aarch64-unknown-linux-musl    | cut -f1 -d' ')
          export MACOS_ARM64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-aarch64-apple-darwin          | cut -f1 -d' ')

          mkdir -p "${ROOT}/release/"
          cat "${ROOT}/ci/templates/manifest.yaml" | envsubst > "${ROOT}/release/manifest.yaml"
      
      - uses: actions/upload-artifact@v4
        with:
          name: manifest.yaml
          path: release/manifest.yaml
          if-no-files-found: error

      - name: Build connector definition
        run: |
          set -evo pipefail
          ROOT="$(pwd)"
          
          export DOCKER_IMAGE="ghcr.io/hasura/ndc-clickhouse:$GITHUB_REF_NAME"
          export CLI_VERSION=$GITHUB_REF_NAME

          mkdir -p "${ROOT}/release/connector-definition/.hasura-connector/"
          cat "${ROOT}/ci/templates/connector-metadata.yaml" | envsubst > "${ROOT}/release/connector-definition/.hasura-connector/connector-metadata.yaml"

          cargo run --package ndc-clickhouse-cli -- --connector-context-path "${ROOT}/release/connector-definition" init

          tar -czvf "${ROOT}/release/artifacts/connector-definition.tgz" --directory "${ROOT}/release/connector-definition/" .

      - uses: actions/upload-artifact@v4
        with:
          name: connector-definition.tgz
          path: ./release/artifacts/connector-definition.tgz
          compression-level: 0 # Already compressed

      - name: Get version from tag
        id: get-version
        run: |
          echo "tagged_version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
        shell: bash

      - uses: mindsers/changelog-reader-action@v2
        id: changelog-reader
        with:
          version: ${{ steps.get-version.outputs.tagged_version }}
          path: ./CHANGELOG.md

      - name: create a release
        uses: ncipollo/release-action@v1
        with:
          draft: false
          tag: v${{ steps.get-version.outputs.tagged_version }}
          body: ${{ steps.changelog-reader.outputs.changes }}
          artifacts: release/artifacts/*
  create-cli-plugins-index-pr:
    name: Create CLI Plugin Index PR
    needs:
      - release
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ secrets.HASURA_BOT_TOKEN }}
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    steps:
      - name: check out this repository
        uses: actions/checkout@v4
        with:
          path: ndc-clickhouse

      - name: check out cli-plugins-index
        uses: actions/checkout@v4
        with:
          repository: hasura/cli-plugins-index
          path: cli-plugins-index
          token: ${{ secrets.HASURA_BOT_TOKEN }}

      - uses: actions/download-artifact@v4
        with:
          path: release/artifacts
          merge-multiple: true

      - name: create cli-plugins-index PR
        run: |
          set -evo pipefail
          ROOT="$(pwd)"

          # we assume GITHUB_REF_NAME is the version number prefixed with `v`. This is checked in previous steps
          BRANCH_NAME="clickhouse/release-$GITHUB_REF_NAME"

          RELEASE_HASH="$(cd ndc-clickhouse && git rev-parse HEAD)"
          
          # Change working directory to the target folder
          cd cli-plugins-index

          git config --global user.name "hasura-bot"
          git config --global user.email "accounts@hasura.io"

          # Create a new feature branch for the changes.
          git checkout -b $BRANCH_NAME

          mkdir "plugins/clickhouse/$GITHUB_REF_NAME"

          cp "${ROOT}/release/artifacts/manifest.yaml" "${ROOT}/cli-plugins-index/plugins/clickhouse/$GITHUB_REF_NAME/manifest.yaml"

          git add .
          git commit -m "Release ClickHouse $GITHUB_REF_NAME"
          git push origin $BRANCH_NAME --force

          # create a pull-requests containing the updates.
          gh pr create \
            --body "Commit in ndc-clickhouse: https://github.com/hasura/ndc-clickhouse/commit/$RELEASE_HASH" \
            --title "Release ClickHouse $GITHUB_REF_NAME" \
            --head "$BRANCH_NAME" \
            --base "master"

  create-ndc-hub-pr:
    name: Create NDC-Hub PR
    needs:
      - release
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ secrets.HASURA_BOT_TOKEN }}
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    steps:
      - name: check out this repository
        uses: actions/checkout@v4
        with:
          path: ndc-clickhouse

      - name: check out ndc-hub
        uses: actions/checkout@v4
        with:
          repository: hasura/ndc-hub
          path: ndc-hub
          token: ${{ secrets.HASURA_BOT_TOKEN }}

      - uses: actions/download-artifact@v4
        with:
          path: release/artifacts
          merge-multiple: true

      - name: create ndc-hub PR
        run: |
          set -evo pipefail
          ROOT="$(pwd)"

          # we assume GITHUB_REF_NAME is the version number prefixed with `v`. This is checked in previous steps
          BRANCH_NAME="clickhouse/release-$GITHUB_REF_NAME"

          # These exported env vars are used for templating
          # Release version is the same as tag name.
          export RELEASE_VERSION="${GITHUB_REF_NAME}"
          export RELEASE_HASH="$(cd ndc-clickhouse && git rev-parse HEAD)"
          export CONNECTOR_DEFINITION_HASH=$(sha256sum ${ROOT}/release/artifacts/connector-definition.tgz | cut -f1 -d' ')
          
          # Change working directory to the target folder
          cd ndc-hub

          git config --global user.name "hasura-bot"
          git config --global user.email "accounts@hasura.io"

          # Create a new feature branch for the changes.
          git checkout -b $BRANCH_NAME

          # create new connector definition
          mkdir "${ROOT}/ndc-hub/registry/hasura/clickhouse/releases/v$RELEASE_VERSION"
          cat "${ROOT}/ndc-clickhouse/ci/templates/connector-packaging.json" | envsubst > "${ROOT}/ndc-hub/registry/hasura/clickhouse/releases/$RELEASE_VERSION/connector-packaging.json"

          # modify metadata file to add new entry
          UPDATED_METADATA= jq --arg RELEASE_VERSION "$RELEASE_VERSION" '.overview.latest_version = $RELEASE_VERSION' "${ROOT}/ndc-hub/registry/hasura/clickhouse/metadata.json" |
            jq --arg RELEASE_VERSION "$RELEASE_VERSION" --arg RELEASE_HASH "$RELEASE_HASH" '.source_code.version |= [{tag: $RELEASE_VERSION, hash: $RELEASE_HASH, is_verified: true}] + .' > metadata.tmp.json

          mv metadata.tmp.json "${ROOT}/ndc-hub/registry/hasura/clickhouse/metadata.json"

          git add .
          git commit -m "Release ClickHouse $RELEASE_VERSION"
          git push origin $BRANCH_NAME --force

          # create a pull-requests containing the updates.
          gh pr create \
            --body "Commit in ndc-clickhouse: https://github.com/hasura/ndc-clickhouse/commit/$RELEASE_HASH" \
            --title "Release ClickHouse $RELEASE_VERSION" \
            --head "$BRANCH_NAME" \
            --base "main"