Remove redundant api dependencies in runtimes base #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2023 Iguazio | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| name: CI | |
| on: | |
| pull_request: | |
| branches: | |
| - development | |
| - '[0-9]+.[0-9]+.x' | |
| - 'feature/**' | |
| # Run CI also on push to backport release branches - we sometimes push code there by cherry-picking, meaning it | |
| # doesn't go through CI (no PR) | |
| # For master all code is coming from development where it's coming from PRs (meaning it's passing CI) but there's the | |
| # rare scenario where the combination of merged PRs won't pass CI | |
| push: | |
| branches: | |
| - master | |
| - '[0-9]+.[0-9]+.x' | |
| env: | |
| NAMESPACE: mlrun-integ-test | |
| jobs: | |
| lint: | |
| name: Lint code (Python ${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| # 3.9 is the current >= 1.3.0 python version | |
| python-version: [3.9] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: pip | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip~=22.3.0 | |
| pip install -r dev-requirements.txt | |
| - name: Lint | |
| run: make lint | |
| go-lint: | |
| name: Lint code (GO) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Dump runner context | |
| run: echo "$RUNNER_CONTEXT" | |
| env: | |
| RUNNER_CONTEXT: ${{ toJson(runner) }} | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version-file: "go/go.mod" | |
| cache-dependency-path: "go/go.sum" | |
| cache: true | |
| - name: Install protobuf compiler | |
| run: | | |
| sudo apt-get update && sudo apt install -y protobuf-compiler | |
| # version is coupled to `go/cmd/schemas_compiler/docker/Dockerfile` | |
| go install google.golang.org/protobuf/cmd/[email protected] | |
| go install google.golang.org/grpc/cmd/[email protected] | |
| - name: Lint | |
| run: make lint-go | |
| tests: | |
| name: Run Dockerized Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # TODO: re-use the next 2 actions instead of duplicating | |
| - name: Extract git branch | |
| id: git_info | |
| run: | | |
| echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT | |
| - name: Resolve docker cache tag | |
| id: docker_cache | |
| run: | | |
| export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.') | |
| export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi) | |
| echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT | |
| - name: Run Dockerized tests | |
| run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-dockerized | |
| integration-tests: | |
| name: Run Dockerized Integration Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # TODO: re-use the next 2 actions instead of duplicating | |
| - name: Extract git branch | |
| id: git_info | |
| run: | | |
| echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT | |
| - name: Resolve docker cache tag | |
| id: docker_cache | |
| run: | | |
| export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.') | |
| export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi) | |
| echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT | |
| - name: Run Dockerized tests | |
| run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-integration-dockerized | |
| go-unit-tests: | |
| name: Run GO Unit Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Run GO tests | |
| run: | | |
| make test-go-unit-dockerized | |
| go-integration-tests: | |
| name: Run GO Integration Tests in Kubernetes Environment | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # since github-actions gives us 14G only, and fills it up with some garbage | |
| - name: Freeing up disk space | |
| run: | | |
| "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh" | |
| - uses: manusa/[email protected] | |
| with: | |
| minikube version: "v1.28.0" | |
| kubernetes version: "v1.25.3" | |
| driver: docker | |
| github token: ${{ github.token }} | |
| - name: Prepare minikube | |
| run: | | |
| minikube kubectl -- config view --flatten > kubeconfig_flatten | |
| echo "MLRUN_TEST_KUBECONFIG=$(pwd)/kubeconfig_flatten" >> $GITHUB_ENV | |
| - name: Run GO tests | |
| run: | | |
| make test-go-integration-dockerized | |
| - name: Output some logs in case of failure | |
| if: ${{ failure() }} | |
| # add set -x to print commands before executing to make logs reading easier | |
| run: | | |
| set -x | |
| minikube ip | |
| minikube logs | |
| minikube kubectl -- --namespace ${NAMESPACE} get events | |
| minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1 | |
| minikube kubectl -- --namespace ${NAMESPACE} get pods | |
| minikube kubectl -- --namespace ${NAMESPACE} get pods -o yaml | |
| minikube kubectl -- --namespace ${NAMESPACE} describe pods | |
| set +x | |
| migrations-tests: | |
| name: Run Dockerized Migrations Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # TODO: re-use the next 2 actions instead of duplicating | |
| - name: Extract git branch | |
| id: git_info | |
| run: | | |
| echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT | |
| - name: Resolve docker cache tag | |
| id: docker_cache | |
| run: | | |
| export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.') | |
| export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi) | |
| echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT | |
| - name: Run Dockerized DB Migration tests | |
| run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-migrations-dockerized | |
| package-tests: | |
| name: Run package tests (Python ${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| # 3.9 is the current >= 1.3.0 python version | |
| # 3.7 is for clients with the previous supported python version | |
| python-version: [3.7, 3.9] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| - name: Install automation scripts dependencies and add mlrun to dev packages | |
| run: pip install -r automation/requirements.txt && pip install -e . | |
| - name: Test package | |
| run: MLRUN_PYTHON_VERSION=${{ matrix.python-version }} make test-package | |
| docs: | |
| name: Build Project Documentation | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # TODO: re-use the next 2 actions instead of duplicating | |
| - name: Extract git branch | |
| id: git_info | |
| run: | | |
| echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT | |
| - name: Resolve docker cache tag | |
| id: docker_cache | |
| run: | | |
| export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.') | |
| export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi) | |
| echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT | |
| - name: Generate HTML docs | |
| run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make html-docs-dockerized | |
| - name: Upload generated docs | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: docs | |
| path: docs/_build/html | |
| backward-compatibility-tests: | |
| name: Run Backward Compatibility Tests | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| env: | |
| MLRUN_BC_TESTS_HOME_PATH: /home/runner/work/mlrun/mlrun | |
| MLRUN_BC_TESTS_BASE_CODE_PATH: /home/runner/work/mlrun/mlrun/base/mlrun | |
| MLRUN_BC_TESTS_HEAD_CODE_PATH: /home/runner/work/mlrun/mlrun/head/mlrun | |
| MLRUN_DOCKER_REGISTRY: ghcr.io/ | |
| steps: | |
| - id: resolve_base_branch | |
| run: | | |
| echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT | |
| - name: Checkout PR Base (target) Branch | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ steps.resolve_base_branch.outputs.branch }} | |
| path: base/mlrun | |
| - name: Checkout Merge Commit (requested branch merged with the target branch) | |
| uses: actions/checkout@v3 | |
| with: | |
| path: head/mlrun | |
| - name: Resolve docker cache tag | |
| id: docker_cache | |
| run: | | |
| export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.') | |
| export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi) | |
| echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT | |
| - name: Run Backward Compatibility Tests | |
| run: | | |
| cd head/mlrun | |
| MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-backward-compatibility-dockerized | |
| check-copyright: | |
| name: Check Copyright Existence | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - name: Check | |
| run: | | |
| deno run --allow-read https://deno.land/x/[email protected]/main.ts | |
| - name: Instructions For Adding Copyright | |
| if: ${{ failure() }} | |
| run: | | |
| echo " | |
| Please add the following lines to the top of your code: | |
| # Copyright 2023 Iguazio | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| " | |
| verify-compatible-requirements-one-after-one: | |
| name: Verify compatible requirements (one after one) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| # 3.9 is the current >= 1.3.0 python version | |
| # 3.7 is for clients with the previous supported python version | |
| python-version: [ 3.7, 3.9 ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| - name: Install requirements ${{ matrix.python-version }} | |
| run: make install-requirements | |
| - name: Install Complete Requirements ${{ matrix.python-version }} | |
| run: make install-complete-requirements | |
| - name: Install All Requirements ${{ matrix.python-version }} | |
| run: make install-all-requirements |