diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala index 37a0c0e99c..44d88bafbe 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala @@ -53,20 +53,11 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa gcLogDir: String, logDir: String ): Array[String] = { - val userEngineResource = engineConnBuildRequest.engineResource - val darResource = userEngineResource.getLockedResource.asInstanceOf[DriverAndYarnResource] val properties = engineConnBuildRequest.engineConnCreationDesc.properties - - val className = getValueAndRemove(properties, "className", mainClass) - val driverCores = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_CORES) - val driverMemory = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_MEMORY) - val executorCores = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_CORES) - val executorMemory = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_MEMORY) - val numExecutors = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_INSTANCES) - val sparkcsonf = getValueAndRemove(properties, LINKIS_SPARK_CONF) + val sparkConf = getValueAndRemove(properties, LINKIS_SPARK_CONF) // sparkcsonf DEMO:spark.sql.shuffle.partitions=10;spark.memory.fraction=0.6 - if (StringUtils.isNotBlank(sparkcsonf)) { - val strArrary = sparkcsonf.split(";").toList + if (StringUtils.isNotBlank(sparkConf)) { + val strArrary = sparkConf.split(";").toList strArrary.foreach { keyAndValue => val key = keyAndValue.split("=")(0).trim val value = keyAndValue.split("=")(1).trim @@ -77,7 +68,15 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa } } } + val className = getValueAndRemove(properties, "className", mainClass) + val driverCores = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_CORES) + val driverMemory = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_MEMORY) + val executorCores = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_CORES) + val executorMemory = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_MEMORY) + val numExecutors = getValueAndRemove(properties, LINKIS_SPARK_EXECUTOR_INSTANCES) + val userEngineResource = engineConnBuildRequest.engineResource + val darResource = userEngineResource.getLockedResource.asInstanceOf[DriverAndYarnResource] val files = getValueAndRemove(properties, "files", "").split(",").filter(isNotBlankPath) val jars = new ArrayBuffer[String]() jars ++= getValueAndRemove(properties, "jars", "").split(",").filter(isNotBlankPath)