add

apache · Dec 16, 2024 · 93ac740 · 93ac740
1 parent d782bd9
commit 93ac740
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 0 deletions.
diff --git a/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy b/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy
@@ -1192,12 +1192,15 @@ suite('test_ms_api', 'p0, docker') {
             def feNode5 = [cloud_unique_id: "${cloudUniqueId}", ip : "${ip5}", edit_log_port: "${edit_log_port}", node_type:"FE_OBSERVER"]
             def feNode6 = [cloud_unique_id: "${cloudUniqueId}", ip : "${ip6}", edit_log_port: "${edit_log_port}", node_type:"FE_FOLLOWER"]
             def addNodesClusterFailed = [cluster_name: "${feClusterName}", cluster_id: "${feClusterId}", type: "SQL", nodes: [feNode4, feNode5, feNode6]]
+            def dropAllFeNodesFailed = [cluster_name: "${feClusterName}", cluster_id: "${feClusterId}", type: "SQL", nodes: [feNodeMap1, feNodeMap2, feNodeMap3, feNode4, feNode5, feNode6]]
             def addNodesClusterSucc = [cluster_name: "${feClusterName}", cluster_id: "${feClusterId}", type: "SQL", nodes: [feNode5, feNode6]]
             def addNodesFailedBody = [instance_id: "${instance_id}", cluster: addNodesClusterFailed]
+            def dropAllFeNodesClusterBody = [instance_id: "${instance_id}", cluster: dropAllFeNodesFailed]
             def addNodesBodySuccBody= [instance_id: "${instance_id}", cluster: addNodesClusterSucc]
             jsonOutput = new JsonOutput()
             def addSomeFENodesFailed = jsonOutput.toJson(addNodesFailedBody)
             def addSomeFENodesSucc = jsonOutput.toJson(addNodesBodySuccBody)
+            def dropAllFeNodesFailedJson = jsonOutput.toJson(dropAllFeNodesClusterBody)
 
             add_node_api.call(msHttpPort, addSomeFENodesFailed) {
                 respCode, body ->
@@ -1225,6 +1228,27 @@ suite('test_ms_api', 'p0, docker') {
                 }
             }
 
+            // inject point, to change MetaServiceImpl_get_cluster_set_config
+            inject_to_ms_api.call(msHttpPort, "resource_manager::set_safe_drop_time", -1) {
+            respCode, body ->
+                log.info("inject resource_manager::set_safe_drop_time resp: ${body} ${respCode}".toString()) 
+            }
+
+            enable_ms_inject_api.call(msHttpPort) {
+                respCode, body ->
+                log.info("enable inject resp: ${body} ${respCode}".toString()) 
+            }
+
+            if (i == 1) {
+                drop_node_api.call(msHttpPort, dropAllFeNodesFailedJson) {
+                    respCode, body ->
+                        json = parseJson(body)
+                        log.info("drop all fe nodes failed http cli result: ${body} ${respCode} ${json}".toString())
+                        assertTrue(json.code.equalsIgnoreCase("INTERNAL_ERROR"))
+                        assertTrue(json.msg.contains("instance invalid, cant modify, plz check")) 
+                }
+            }
+
             get_instance_api.call(msHttpPort, instance_id) {
                 respCode, body ->
                     log.info("add Master-observer mode get instance resp: ${body} ${respCode}".toString())

diff --git a/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy b/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy
@@ -36,6 +36,7 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') {
         ]
         options.cloudMode = true
         options.sqlModeNodeMgr = true
+        options.connectToFollower = true
         options.waitTimeout = 0
         options.feNum = 3
         options.feConfigs += ["resource_not_ready_sleep_seconds=1",
@@ -58,9 +59,48 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') {
     clusterOptions[3].beClusterId = false;
     clusterOptions[3].beMetaServiceEndpoint = false;
 
+    def inject_to_ms_api = { msHttpPort, key, value, check_func ->
+        httpTest {
+            op "get"
+            endpoint msHttpPort
+            uri "/MetaService/http/v1/injection_point?token=${token}&op=set&name=${key}&behavior=set&value=${value}"
+            check check_func
+        }
+    }
+
+    def clear_ms_inject_api = { msHttpPort, key, value, check_func ->
+        httpTest {
+            op "get"
+            endpoint msHttpPort
+            uri "/MetaService/http/v1/injection_point?token=${token}&op=clear"
+            check check_func
+        }
+    }
+
+    def enable_ms_inject_api = { msHttpPort, check_func ->
+        httpTest {
+            op "get"
+            endpoint msHttpPort
+            uri "/MetaService/http/v1/injection_point?token=${token}&op=enable"
+            check check_func
+        }
+    }
+
     for (options in clusterOptions) {
         docker(options) {
             logger.info("docker started");
+            def ms = cluster.getAllMetaservices().get(0)
+            def msHttpPort = ms.host + ":" + ms.httpPort
+            // inject point, to change MetaServiceImpl_get_cluster_set_config
+            inject_to_ms_api.call(msHttpPort, "resource_manager::set_safe_drop_time", -1) {
+            respCode, body ->
+                log.info("inject resource_manager::set_safe_drop_time resp: ${body} ${respCode}".toString()) 
+            }
+
+            enable_ms_inject_api.call(msHttpPort) {
+                respCode, body ->
+                log.info("enable inject resp: ${body} ${respCode}".toString()) 
+            }
 
             def checkFrontendsAndBackends = {
                 // Check frontends
@@ -489,6 +529,7 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') {
             logger.info("Successfully decommissioned backend and verified its status")
 
             checkClusterStatus(3, 3, 8)
+
         }
     }