Skip to content

[FLINK-36129] Supporting Flink 1.20 and removing Flink 1.15 #1

[FLINK-36129] Supporting Flink 1.20 and removing Flink 1.15

[FLINK-36129] Supporting Flink 1.20 and removing Flink 1.15 #1

Workflow file for this run

################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
# We need to specify repo related information here since Apache INFRA doesn't differentiate
# between several workflows with the same names while preparing a report for GHA usage
# https://infra-reports.apache.org/#ghactions
name: Flink Kubernetes Operator CI
on:
push:
pull_request:
jobs:
test_ci:
runs-on: ubuntu-latest
name: test_ci
strategy:
matrix:
java-version: [ 11, 17, 21 ]
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java-version }}
uses: actions/setup-java@v2
with:
java-version: ${{ matrix.java-version }}
distribution: 'adopt'
- name: Cache local Maven repository
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Build with Maven
run: |
set -o pipefail; mvn clean install javadoc:javadoc -Pgenerate-docs | tee ./mvn.log; set +o pipefail
if [[ $(cat ./mvn.log | grep -E -v '(flink-runtime-.*.jar, flink-kubernetes-operator-.*.jar)|(flink-kubernetes-operator-.*.jar, flink-runtime-.*.jar) define 3 overlapping classes' | grep -E -v '(flink-runtime-.*.jar, flink-autoscaler-.*.jar)|(flink-autoscaler-.*.jar, flink-runtime-.*.jar) define 1|2 overlapping classes' | grep -c "overlapping classes" -) -gt 0 ]];then
echo "Found overlapping classes: "
cat ./mvn.log | grep "overlapping classes"
exit 1
fi
if [[ $(git diff HEAD | wc -l) -gt 0 ]];then
echo "Please generate the java doc via 'mvn clean install -DskipTests -Pgenerate-docs' again"
exit 1
fi
- name: Validate helm chart linting
run: |
helm lint helm/flink-kubernetes-operator
- name: Tests in flink-kubernetes-operator
run: |
cd flink-kubernetes-operator
mvn verify -Dit.skip=false
cd ..
- name: Tests in flink-kubernetes-webhook
run: |
cd flink-kubernetes-webhook
mvn verify -Dit.skip=false
cd ..
- name: Tests in flink-autoscaler-plugin-jdbc
run: |
cd flink-autoscaler-plugin-jdbc
mvn verify -Dit.skip=false
cd ..
e2e_ci:
runs-on: ubuntu-latest
strategy:
matrix:
version: ["v1_20","v1_19","v1_18","v1_17","v1_16"]
namespace: ["default","flink"]
mode: ["native", "standalone"]
java-version: [ 11, 17, 21 ]
test:
- test_application_kubernetes_ha.sh
- test_application_operations.sh
- test_dynamic_config.sh
- test_sessionjob_kubernetes_ha.sh
- test_sessionjob_operations.sh
- test_multi_sessionjob.sh
- test_autoscaler.sh
- test_flink_operator_ha.sh
include:
- namespace: flink
extraArgs: '--create-namespace --set "watchNamespaces={default,flink}"'
- version: v1_20
image: flink:1.20
- version: v1_19
image: flink:1.19
- version: v1_18
image: flink:1.18
- version: v1_17
image: flink:1.17
- version: v1_16
image: flink:1.16
exclude:
- namespace: default
test: test_multi_sessionjob.sh
- namespace: default
test: test_autoscaler.sh
- namespace: default
test: test_dynamic_config.sh
- mode: standalone
test: test_autoscaler.sh
- mode: standalone
test: test_dynamic_config.sh
- version: v1_16
test: test_autoscaler.sh
- version: v1_16
test: test_dynamic_config.sh
- version: v1_16
test: test_flink_operator_ha.sh
- version: v1_17
test: test_dynamic_config.sh
- version: v1_17
test: test_flink_operator_ha.sh
- version: v1_18
test: test_dynamic_config.sh
- version: v1_18
test: test_flink_operator_ha.sh
- version: v1_16
java-version: 17
- version: v1_17
java-version: 17
- version: v1_16
java-version: 21
- version: v1_17
java-version: 21
- version: v1_18
java-version: 21
name: e2e_ci
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java-version }}
uses: actions/setup-java@v2
with:
java-version: ${{ matrix.java-version }}
distribution: 'adopt'
- name: Cache local Maven repository
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Start minikube
run: |
source e2e-tests/utils.sh
start_minikube
- name: Install cert-manager
run: |
kubectl get pods -A
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
- name: Build image
run: |
export SHELL=/bin/bash
export DOCKER_BUILDKIT=1
eval $(minikube -p minikube docker-env)
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain .
docker images
- name: Start the operator
run: |
if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then
sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
fi
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{ matrix.extraArgs }}
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
kubectl get pods -n ${{ matrix.namespace }}
- name: Run Flink e2e tests
run: |
sed -i "s/image: flink:.*/image: ${{ matrix.image }}/" e2e-tests/data/*.yaml
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.version }}/" e2e-tests/data/*.yaml
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
git diff HEAD
echo "Running e2e-tests/$test"
bash e2e-tests/${{ matrix.test }} || exit 1
git reset --hard
- name: Stop the operator
run: |
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
- name: Stop minikube
run: |
source e2e-tests/utils.sh
stop_minikube