【1.1.15】spark code opmitization (#287)

* spark conf * Remove spaces * code opmitization --------- Co-authored-by: huangKai-2323 <[email protected]>
WeDataSphere · Sep 11, 2023 · a95b477 · a95b477
1 parent 4960c3d
commit a95b477
Showing 1 changed file with 11 additions and 12 deletions.
diff --git a/...g/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala b/...g/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala
@@ -53,20 +53,11 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa
       gcLogDir: String,
       logDir: String
   ): Array[String] = {
-    val userEngineResource = engineConnBuildRequest.engineResource
-    val darResource = userEngineResource.getLockedResource.asInstanceOf[DriverAndYarnResource]
     val properties = engineConnBuildRequest.engineConnCreationDesc.properties
-
-    val className = getValueAndRemove(properties, "className", mainClass)
-    val driverCores = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_CORES)
-    val driverMemory = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_MEMORY)
-    val executorCores = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_CORES)
-    val executorMemory = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_MEMORY)
-    val numExecutors = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_INSTANCES)
-    val sparkcsonf = getValueAndRemove(properties, LINKIS_SPARK_CONF)
+    val sparkConf = getValueAndRemove(properties, LINKIS_SPARK_CONF)
     // sparkcsonf DEMO:spark.sql.shuffle.partitions=10;spark.memory.fraction=0.6
-    if (StringUtils.isNotBlank(sparkcsonf)) {
-      val strArrary = sparkcsonf.split(";").toList
+    if (StringUtils.isNotBlank(sparkConf)) {
+      val strArrary = sparkConf.split(";").toList
       strArrary.foreach { keyAndValue =>
         val key = keyAndValue.split("=")(0).trim
         val value = keyAndValue.split("=")(1).trim
@@ -77,7 +68,15 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa
         }
       }
     }
+    val className = getValueAndRemove(properties, "className", mainClass)
+    val driverCores = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_CORES)
+    val driverMemory = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_MEMORY)
+    val executorCores = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_CORES)
+    val executorMemory = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_MEMORY)
+    val numExecutors = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_INSTANCES)
 
+    val userEngineResource = engineConnBuildRequest.engineResource
+    val darResource = userEngineResource.getLockedResource.asInstanceOf[DriverAndYarnResource]
     val files = getValueAndRemove(properties, "files", "").split(",").filter(isNotBlankPath)
     val jars = new ArrayBuffer[String]()
     jars ++= getValueAndRemove(properties, "jars", "").split(",").filter(isNotBlankPath)