slow_tests #52
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: slow_tests | |
| on: | |
| # The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags. | |
| create: | |
| tags: | |
| - "v*.*.*" # normal release | |
| - "nightly" # mutable tag | |
| - "slow-test" # mutable tag | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| slow_tests: | |
| name: run slow test | |
| runs-on: ["self-hosted", "slow-test" ] | |
| steps: | |
| - name: Show PR labels | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
| fi | |
| - name: Ensure workspace ownership | |
| if: ${{ !cancelled() && !failure() }} | |
| run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
| - name: Check out code | |
| if: ${{ !cancelled() && !failure() }} | |
| uses: actions/checkout@v3 | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| CPUS=${CPUS:-$(nproc)} | |
| echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV | |
| echo "CPUS=${CPUS}" >> $GITHUB_ENV | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18 | |
| - name: Build release version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target infinity test_main knn_import_benchmark knn_query_benchmark" | |
| - name: Install pysdk for Python 3.10 | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
| - name: Prepare restart test data | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME} | |
| echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV | |
| touch tmp.txt && echo "${RUNNER_WORKSPACE_PREFIX}" > tmp.txt | |
| sudo mkdir -p test/data/benchmark && sudo ln -sf ${RUNNER_WORKSPACE_PREFIX}/benchmark/enwiki/enwiki-10w.csv test/data/benchmark/enwiki-10w.csv | |
| - name: Run restart test | |
| if: ${{ !cancelled() && !failure() }} | |
| run : | | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_data[infinity_runner0-columns5-gen-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity" | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_index[infinity_runner0-columns2-indexes2-gen-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity" | |
| - name: Collect restart test output | |
| if: ${{ !cancelled() }} # always run this step even if previous steps failed | |
| # remove symbolic link | |
| # find all log file like [debug.log.*] in directory, and cat to stdout | |
| run: | | |
| sudo rm -f test/data/benchmark/enwiki-10w.csv | |
| find . -name "restart_test.log.*" -exec cat {} \; | |
| - name: Destroy builder container | |
| if: always() # always run this step even if previous steps failed | |
| run: | | |
| if [ -n "${BUILDER_CONTAINER}" ]; then | |
| sudo docker rm -f -v ${BUILDER_CONTAINER} | |
| fi | |