Fix the error of vineyardctl and add the related test.

.github/workflows/vineyard-operator.yaml

@@ -74,6 +74,7 @@ jobs:
           - e2e-tests-workflow
           - e2e-tests-mars-examples
           - e2e-tests-vineyardctl
+          - e2e-tests-vineyardctl-python-api
           - e2e-tests-deploy-raw-backup-and-recover
           - e2e-tests-three-etcd-nodes-failover
           - e2e-tests-five-etcd-nodes-failover
@@ -161,6 +162,7 @@ jobs:
 
       - name: Generate the python and vineyardd image for tests
         run: |
+          go build -a -o python/vineyard/bdist/vineyardctl k8s/cmd/main.go && \
           make -C k8s/test/e2e build-base-images
 
       - name: unit-tests
@@ -278,6 +280,11 @@ jobs:
         if: ${{ matrix.job == 'e2e-tests-vineyardctl' }}
         run: |
           make -C k8s/test/e2e e2e-tests-vineyardctl
+
+      - name: e2e-tests-vineyardctl-python-api
+        if: ${{ matrix.job == 'e2e-tests-vineyardctl-python-api' }}
+        run: |
+          make -C k8s/test/e2e e2e-tests-vineyardctl-python-api
 
       - name: e2e-tests-deploy-raw-backup-and-recover
         if: ${{ matrix.job == 'e2e-tests-deploy-raw-backup-and-recover' }}

k8s/cmd/commands/util/usage/types.go

@@ -59,10 +59,14 @@ func CobraUsage(cmd *cobra.Command) (Command, error) {
 		c.Children = append(c.Children, childUsage)
 	}
 	cmd.LocalFlags().VisitAll(func(flag *pflag.Flag) {
-		c.Flags = append(c.Flags, FlagUsage(flag))
+		if flag.Name != "log-flush-frequency" {
+			c.Flags = append(c.Flags, FlagUsage(flag))
+		}
 	})
 	cmd.InheritedFlags().VisitAll(func(flag *pflag.Flag) {
-		c.GlobalFlags = append(c.GlobalFlags, FlagUsage(flag))
+		if flag.Name != "log-flush-frequency" {
+			c.GlobalFlags = append(c.GlobalFlags, FlagUsage(flag))
+		}
 	})
 	return c, nil
 }

k8s/cmd/main.go

@@ -63,6 +63,11 @@ func init() {
 
 	flags.ApplyGlobalFlags(cmd)
 
+	if cmd.Flags().Lookup("log-flush-frequency") != nil {
+		if err := cmd.Flags().MarkHidden("log-flush-frequency"); err != nil {
+			log.Fatal(err, "Failed to mark hidden flag")
+		}
+	}
 	// disable completion command
 	cmd.CompletionOptions.DisableDefaultCmd = true
 

k8s/test/e2e/Makefile

@@ -384,6 +384,16 @@ e2e-tests-vineyardctl: prepare-e2e-test build-local-cluster load-vineyardd-image
 	@make delete-local-cluster
 .PHONY: e2e-tests-vineyardctl
 
+############# vineyardctl python api testing ################################################
+
+e2e-tests-vineyardctl-python-api: prepare-e2e-test build-local-cluster load-vineyardd-image load-vineyard-python-dev-image
+	@make -C ${E2E_DIR} install-vineyard-operator
+	@echo "Running vineyardctl python api e2e test..."
+	@cd ${ROOT_DIR} && ${GOBIN}/e2e run --config=${E2E_DIR}/vineyardctl-python-api/e2e.yaml
+	@echo "vineyardctl python api e2e test passed."
+	@make delete-local-cluster
+.PHONY: e2e-tests-vineyardctl-python-api
+
 ############# backup and recover testing ################################################
 
 e2e-tests-deploy-raw-backup-and-recover: prepare-e2e-test build-local-cluster load-vineyardd-image

k8s/test/e2e/etcd-failover/three-etcd-nodes-failover-e2e.yaml

@@ -32,7 +32,7 @@ setup:
           kubectl delete pod vineyardd-$(shuf -i 0-2 -n 1) -n default --force
           kubectl rollout status statefulset/vineyardd
           # wait for the instance quit messages to be propagated
-          sleep 120
+          sleep 180
           kubectl rollout status statefulset/vineyardd
         done
     - name: install consumer

k8s/test/e2e/vineyardctl-python-api/e2e.yaml

@@ -0,0 +1,70 @@
+# Copyright 2020-2023 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+setup:
+  env: kind
+  kubeconfig: /tmp/e2e-k8s.config
+  steps:
+    - name: download all workflow images into kind cluster
+      command: |
+        make -C k8s/test/e2e publish-workflow-images REGISTRY=localhost:5001
+    - name: install vineyardd
+      command: |
+        docker run --net=host -v /tmp/e2e-k8s.config:/root/.kube/config:ro localhost:5001/vineyard-python-dev:latest \
+          "python -c \"import vineyard; vineyard.deploy.vineyardctl.deploy.vineyardd(vineyardd_image='localhost:5001/vineyardd:latest', kubeconfig='/root/.kube/config')\""
+    - name: install job1
+      command: |
+        kubectl create namespace vineyard-job
+        kubectl apply -f k8s/test/e2e/workflow-demo/workflow-job1.yaml
+      wait:
+        - namespace: vineyard-job
+          resource: job/v6d-workflow-demo-job1
+          for: condition=Complete
+    - name: install job2
+      command: |
+        kubectl apply -f k8s/test/e2e/workflow-demo/workflow-job2.yaml
+      wait:
+        - namespace: vineyard-job
+          resource: job/v6d-workflow-demo-job2
+          for: condition=Complete
+  timeout: 20m
+
+cleanup:
+  # always never success failure
+  on: never
+
+verify:
+  # verify with retry strategy
+  retry:
+    # max retry count
+    count: 10
+    # the interval between two attempts, e.g. 10s, 1m.
+    interval: 10s
+  cases:
+    - query: |
+        kubectl get pod -l app=v6d-workflow-demo-job2 -n vineyard-job -oname | \
+          awk -F '/' '{print $2}' | \
+          head -n 1 | \
+          xargs kubectl logs -n vineyard-job | \
+          yq e '{"sum": .}' - | \
+          yq e 'to_entries' -
+      expected: ../verify/values.yaml
+    # test the job can only be scheduled on the nodes with the vineyardd
+    - query: |
+        export job1_nodes=$(kubectl get pod -l app=v6d-workflow-demo-job1 -nvineyard-job -o=custom-columns=NODE:.spec.nodeName | awk 'NR != 1' | sort | tr '\n' ' ')
+        export job2_nodes=$(kubectl get pod -l app=v6d-workflow-demo-job2 -nvineyard-job -o=custom-columns=NODE:.spec.nodeName | awk 'NR != 1' | sort | tr '\n' ' ')
+        export vineyardd_nodes=$(kubectl get pod -l app.kubernetes.io/instance=vineyardd -n vineyard-system -o=custom-columns=NODE:.spec.nodeName | awk 'NR != 1' | sort | tr '\n' ' ')
+        if [[ $job1_nodes = $vineyardd_nodes ]] && [[ $job2_nodes = $vineyardd_nodes ]]; then echo '{"AllJobInVineyarddNodes":"true"}' | yq e 'to_entries' -; fi
+      expected: ../verify/nodes.yaml