From c45b59f5423fe8563bfd253f1a023f6afd00b34e Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Sun, 4 Aug 2024 11:40:08 +0800 Subject: [PATCH 1/6] [spark] Support show columns in spark --- .../paimon/spark/sql/ShowColumnsTest.scala | 21 +++++++++ .../paimon/spark/sql/ShowColumnsTest.scala | 21 +++++++++ .../paimon/spark/sql/ShowColumnsTest.scala | 21 +++++++++ .../paimon/spark/sql/ShowColumnsTest.scala | 21 +++++++++ .../catalyst/analysis/PaimonAnalysis.scala | 5 +- .../commands/PaimonShowColumnsCommand.scala | 36 ++++++++++++++ .../execution/PaimonShowColumnsExec.scala | 34 ++++++++++++++ .../spark/execution/PaimonStrategy.scala | 4 ++ .../spark/sql/PaimonShowColumnsTestBase.scala | 47 +++++++++++++++++++ 9 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala create mode 100644 paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala create mode 100644 paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala create mode 100644 paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala create mode 100644 paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala create mode 100644 paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala create mode 100644 paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala diff --git a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala new file mode 100644 index 000000000000..6601dc2fca37 --- /dev/null +++ b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class ShowColumnsTest extends PaimonShowColumnsTestBase {} diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala new file mode 100644 index 000000000000..6601dc2fca37 --- /dev/null +++ b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class ShowColumnsTest extends PaimonShowColumnsTestBase {} diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala new file mode 100644 index 000000000000..6601dc2fca37 --- /dev/null +++ b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class ShowColumnsTest extends PaimonShowColumnsTestBase {} diff --git a/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala new file mode 100644 index 000000000000..6601dc2fca37 --- /dev/null +++ b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class ShowColumnsTest extends PaimonShowColumnsTestBase {} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala index 67685612664d..2a390fd0a115 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala @@ -21,7 +21,7 @@ package org.apache.paimon.spark.catalyst.analysis import org.apache.paimon.spark.SparkTable import org.apache.paimon.spark.catalyst.Compatibility import org.apache.paimon.spark.catalyst.analysis.PaimonRelation.isPaimonTable -import org.apache.paimon.spark.commands.{PaimonAnalyzeTableColumnCommand, PaimonDynamicPartitionOverwriteCommand, PaimonTruncateTableCommand} +import org.apache.paimon.spark.commands.{PaimonAnalyzeTableColumnCommand, PaimonDynamicPartitionOverwriteCommand, PaimonShowColumnsCommand, PaimonTruncateTableCommand} import org.apache.paimon.table.FileStoreTable import org.apache.spark.sql.SparkSession @@ -55,6 +55,9 @@ class PaimonAnalysis(session: SparkSession) extends Rule[LogicalPlan] { case merge: MergeIntoTable if isPaimonTable(merge.targetTable) && merge.childrenResolved => PaimonMergeIntoResolver(merge, session) + + case s @ ShowColumns(PaimonRelation(table), _, _) if s.resolved => + PaimonShowColumnsCommand(table) } private def schemaCompatible( diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala new file mode 100644 index 000000000000..f53b6f66fb7f --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.commands + +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.leafnode.PaimonLeafRunnableCommand +import org.apache.paimon.table.FileStoreTable + +import org.apache.spark.sql.{Row, SparkSession} + +case class PaimonShowColumnsCommand(v2Table: SparkTable) + extends PaimonLeafRunnableCommand + with WithFileStoreTable { + override def table: FileStoreTable = v2Table.getTable.asInstanceOf[FileStoreTable] + + override def run(sparkSession: SparkSession): Seq[Row] = { + Seq.empty[Row] + } + +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala new file mode 100644 index 000000000000..443842b0c29b --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute + +case class PaimonShowColumnsExec(output: Seq[Attribute], v2Table: SparkTable) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + v2Table.columns().map(f => InternalRow.fromSeq(Seq(f.name()))).toSeq + } + +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala index c6c6fc8759c0..f3392fdbba5e 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala @@ -19,6 +19,7 @@ package org.apache.paimon.spark.execution import org.apache.paimon.spark.catalyst.plans.logical.PaimonCallCommand +import org.apache.paimon.spark.commands.PaimonShowColumnsCommand import org.apache.spark.sql.{SparkSession, Strategy} import org.apache.spark.sql.catalyst.InternalRow @@ -37,6 +38,9 @@ case class PaimonStrategy(spark: SparkSession) extends Strategy with PredicateHe case c @ PaimonCallCommand(procedure, args) => val input = buildInternalRow(args) PaimonCallExec(c.output, procedure, input) :: Nil + + case c @ PaimonShowColumnsCommand(table) => + PaimonShowColumnsExec(c.output, table) :: Nil case _ => Nil } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala new file mode 100644 index 000000000000..cfdd696eaa5f --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.junit.jupiter.api.Assertions.assertDoesNotThrow + +class PaimonShowColumnsTestBase extends PaimonSparkTestBase { + + test("Show columns from Paimon test") { + spark.sql(s""" + |CREATE TABLE T (id STRING, name STRING, i INT, l LONG) + |USING PAIMON + |TBLPROPERTIES ('primary-key'='id') + |""".stripMargin) + + spark.sql( + s""" + |INSERT INTO T SELECT '1', 'aa', 12, 22 + |""".stripMargin + ) + + assertDoesNotThrow( + () => { + spark.sql("SHOW COLUMNS FROM T") + }) + + } + +} From 63650885b03a9cbae3aed1d89e99629ed260d951 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Sun, 4 Aug 2024 14:49:03 +0800 Subject: [PATCH 2/6] compabitive --- .../apache/paimon/spark/execution/PaimonShowColumnsExec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala index 443842b0c29b..7fb6f5c38c0e 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala @@ -28,7 +28,7 @@ case class PaimonShowColumnsExec(output: Seq[Attribute], v2Table: SparkTable) extends PaimonLeafV2CommandExec { override protected def run(): Seq[InternalRow] = { - v2Table.columns().map(f => InternalRow.fromSeq(Seq(f.name()))).toSeq + v2Table.schema.map(sc => InternalRow.fromSeq(Seq(sc.name))).toSeq } } From 9f6afb6cff49a14355b778178d1ce92acf62851c Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Mon, 5 Aug 2024 20:34:53 +0800 Subject: [PATCH 3/6] fix ut --- .../spark/commands/PaimonShowColumnsCommand.scala | 11 ++++++----- .../spark/execution/PaimonShowColumnsExec.scala | 3 ++- .../paimon/spark/execution/PaimonStrategy.scala | 4 ++-- .../paimon/spark/sql/PaimonShowColumnsTestBase.scala | 7 ++++++- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala index f53b6f66fb7f..d16b2f67fa65 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala @@ -19,18 +19,19 @@ package org.apache.paimon.spark.commands import org.apache.paimon.spark.SparkTable -import org.apache.paimon.spark.leafnode.PaimonLeafRunnableCommand +import org.apache.paimon.spark.leafnode.{PaimonLeafCommand, PaimonLeafRunnableCommand} import org.apache.paimon.table.FileStoreTable import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} +import org.apache.spark.sql.types.{BinaryType, Metadata, StringType} case class PaimonShowColumnsCommand(v2Table: SparkTable) - extends PaimonLeafRunnableCommand + extends PaimonLeafCommand with WithFileStoreTable { override def table: FileStoreTable = v2Table.getTable.asInstanceOf[FileStoreTable] - override def run(sparkSession: SparkSession): Seq[Row] = { - Seq.empty[Row] - } + override lazy val output: Seq[Attribute] = Seq( + AttributeReference("column", StringType, true, Metadata.empty)()) } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala index 7fb6f5c38c0e..ed17434f4e50 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala @@ -23,12 +23,13 @@ import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.unsafe.types.UTF8String case class PaimonShowColumnsExec(output: Seq[Attribute], v2Table: SparkTable) extends PaimonLeafV2CommandExec { override protected def run(): Seq[InternalRow] = { - v2Table.schema.map(sc => InternalRow.fromSeq(Seq(sc.name))).toSeq + v2Table.schema.map(sc => InternalRow.fromSeq(Seq(UTF8String.fromString(sc.name)))).toSeq } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala index f3392fdbba5e..d34903f6a4fb 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala @@ -39,8 +39,8 @@ case class PaimonStrategy(spark: SparkSession) extends Strategy with PredicateHe val input = buildInternalRow(args) PaimonCallExec(c.output, procedure, input) :: Nil - case c @ PaimonShowColumnsCommand(table) => - PaimonShowColumnsExec(c.output, table) :: Nil + case s @ PaimonShowColumnsCommand(table) => + PaimonShowColumnsExec(s.output, table) :: Nil case _ => Nil } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala index cfdd696eaa5f..3aa0f74fd3d7 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala @@ -20,9 +20,10 @@ package org.apache.paimon.spark.sql import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.spark.sql.Row import org.junit.jupiter.api.Assertions.assertDoesNotThrow -class PaimonShowColumnsTestBase extends PaimonSparkTestBase { +abstract class PaimonShowColumnsTestBase extends PaimonSparkTestBase { test("Show columns from Paimon test") { spark.sql(s""" @@ -42,6 +43,10 @@ class PaimonShowColumnsTestBase extends PaimonSparkTestBase { spark.sql("SHOW COLUMNS FROM T") }) + checkAnswer( + spark.sql("SHOW COLUMNS FROM T"), + Row("id") :: Row("name") :: Row("i") :: Row("l") :: Nil) + } } From 90c47347f695b33e9f1c06af4f90b008ad4ec758 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Mon, 5 Aug 2024 21:48:07 +0800 Subject: [PATCH 4/6] optizme code --- .../commands/PaimonShowColumnsCommand.scala | 10 ++++-- .../execution/PaimonShowColumnsExec.scala | 35 ------------------- .../spark/execution/PaimonStrategy.scala | 3 -- .../spark/sql/PaimonShowColumnsTestBase.scala | 6 ---- 4 files changed, 8 insertions(+), 46 deletions(-) delete mode 100644 paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala index d16b2f67fa65..0c6c11abe205 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala @@ -23,15 +23,21 @@ import org.apache.paimon.spark.leafnode.{PaimonLeafCommand, PaimonLeafRunnableCo import org.apache.paimon.table.FileStoreTable import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, GenericRow} import org.apache.spark.sql.types.{BinaryType, Metadata, StringType} +import org.apache.spark.unsafe.types.UTF8String case class PaimonShowColumnsCommand(v2Table: SparkTable) - extends PaimonLeafCommand +// extends PaimonLeafCommand + extends PaimonLeafRunnableCommand with WithFileStoreTable { override def table: FileStoreTable = v2Table.getTable.asInstanceOf[FileStoreTable] override lazy val output: Seq[Attribute] = Seq( AttributeReference("column", StringType, true, Metadata.empty)()) + override def run(sparkSession: SparkSession): Seq[Row] = { + v2Table.schema.map(sc => new GenericRow(Array[Any](UTF8String.fromString(sc.name)))).toSeq + } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala deleted file mode 100644 index ed17434f4e50..000000000000 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonShowColumnsExec.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.execution - -import org.apache.paimon.spark.SparkTable -import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec - -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.unsafe.types.UTF8String - -case class PaimonShowColumnsExec(output: Seq[Attribute], v2Table: SparkTable) - extends PaimonLeafV2CommandExec { - - override protected def run(): Seq[InternalRow] = { - v2Table.schema.map(sc => InternalRow.fromSeq(Seq(UTF8String.fromString(sc.name)))).toSeq - } - -} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala index d34903f6a4fb..8a1df9622d81 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala @@ -19,7 +19,6 @@ package org.apache.paimon.spark.execution import org.apache.paimon.spark.catalyst.plans.logical.PaimonCallCommand -import org.apache.paimon.spark.commands.PaimonShowColumnsCommand import org.apache.spark.sql.{SparkSession, Strategy} import org.apache.spark.sql.catalyst.InternalRow @@ -39,8 +38,6 @@ case class PaimonStrategy(spark: SparkSession) extends Strategy with PredicateHe val input = buildInternalRow(args) PaimonCallExec(c.output, procedure, input) :: Nil - case s @ PaimonShowColumnsCommand(table) => - PaimonShowColumnsExec(s.output, table) :: Nil case _ => Nil } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala index 3aa0f74fd3d7..961c568e139e 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala @@ -21,7 +21,6 @@ package org.apache.paimon.spark.sql import org.apache.paimon.spark.PaimonSparkTestBase import org.apache.spark.sql.Row -import org.junit.jupiter.api.Assertions.assertDoesNotThrow abstract class PaimonShowColumnsTestBase extends PaimonSparkTestBase { @@ -38,11 +37,6 @@ abstract class PaimonShowColumnsTestBase extends PaimonSparkTestBase { |""".stripMargin ) - assertDoesNotThrow( - () => { - spark.sql("SHOW COLUMNS FROM T") - }) - checkAnswer( spark.sql("SHOW COLUMNS FROM T"), Row("id") :: Row("name") :: Row("i") :: Row("l") :: Nil) From 2a130b8d78aa228751c66719a0e6d062a0c2a462 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Mon, 5 Aug 2024 21:53:14 +0800 Subject: [PATCH 5/6] optizme imports --- .../paimon/spark/commands/PaimonShowColumnsCommand.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala index 0c6c11abe205..1067526f37bc 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowColumnsCommand.scala @@ -19,17 +19,15 @@ package org.apache.paimon.spark.commands import org.apache.paimon.spark.SparkTable -import org.apache.paimon.spark.leafnode.{PaimonLeafCommand, PaimonLeafRunnableCommand} +import org.apache.paimon.spark.leafnode.PaimonLeafRunnableCommand import org.apache.paimon.table.FileStoreTable import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, GenericRow} -import org.apache.spark.sql.types.{BinaryType, Metadata, StringType} +import org.apache.spark.sql.types.{Metadata, StringType} import org.apache.spark.unsafe.types.UTF8String case class PaimonShowColumnsCommand(v2Table: SparkTable) -// extends PaimonLeafCommand extends PaimonLeafRunnableCommand with WithFileStoreTable { override def table: FileStoreTable = v2Table.getTable.asInstanceOf[FileStoreTable] From c0c60a3e7828d46b6d8d5409b3d6ad4c47563378 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Mon, 5 Aug 2024 22:26:52 +0800 Subject: [PATCH 6/6] remove line --- .../scala/org/apache/paimon/spark/execution/PaimonStrategy.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala index 8a1df9622d81..c6c6fc8759c0 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala @@ -37,7 +37,6 @@ case class PaimonStrategy(spark: SparkSession) extends Strategy with PredicateHe case c @ PaimonCallCommand(procedure, args) => val input = buildInternalRow(args) PaimonCallExec(c.output, procedure, input) :: Nil - case _ => Nil }