[docker](iceberg) add iceberg docker compose and modify scripts (apac…

…he#16175)

Add iceberg docker compose
Rename start-thirdparties-docker.sh to run-thirdparties-docker.sh and support start to stop specified components.

.github/workflows/code-checks.yml

@@ -49,7 +49,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           sh_checker_comment: true
-          sh_checker_exclude: .git .github ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$
+          sh_checker_exclude: .git .github ^docker/thirdparties ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$
 
   clang-tidy:
     name: "Clang Tidy"

.licenserc.yaml

@@ -78,5 +78,6 @@ header:
     - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql"
     - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql"
     - "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/"
+    - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
 
   comment: on-failure

docker/thirdparties/custom_settings.env

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Change this to a specific string.
+# Do not use "_" or other sepcial characters, only number and alphabeta.
+# eg: CONTAINER_UID="doris-jack-"
+# NOTICE: change this uid will modify the file in docker-compose.
+CONTAINER_UID="doris--"

docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export SPARK_MASTER_HOST=doris--spark-iceberg
+
+start-master.sh -p 7077
+start-worker.sh spark://doris--spark-iceberg:7077
+start-history-server.sh
+start-thriftserver.sh
+
+# Entrypoint, for example notebook, pyspark or spark-sql
+if [[ $# -gt 0 ]]; then
+    eval "$1"
+fi
+
+# Avoid container exit
+while true; do
+    sleep 1
+done

docker/thirdparties/docker-compose/iceberg/iceberg.env

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+NOTEBOOK_SERVER_PORT=8888
+SPARK_DRIVER_UI_PORT=8080
+SPARK_HISTORY_UI_PORT=10000
+REST_CATALOG_PORT=8181
+MINIO_UI_PORT=9000
+MINIO_API_PORT=9001

docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl

@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+  doris--spark-iceberg:
+    image: tabulario/spark-iceberg
+    container_name: doris--spark-iceberg
+    hostname: doris--spark-iceberg
+    build: spark/
+    depends_on:
+      - doris--rest
+      - doris--minio
+    volumes:
+      - ./warehouse:/home/iceberg/warehouse
+      - ./notebooks:/home/iceberg/notebooks/notebooks
+      - ./entrypoint.sh:/opt/spark/entrypoint.sh
+      - ./spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+    ports:
+      - ${NOTEBOOK_SERVER_PORT}:8888
+      - ${SPARK_DRIVER_UI_PORT}:8080
+      - ${SPARK_HISTORY_UI_PORT}:10000
+    links:
+      - doris--rest:rest
+      - doris--minio:minio
+    networks:
+      - doris--iceberg
+    entrypoint:
+      - /opt/spark/entrypoint.sh
+
+  doris--rest:
+    image: tabulario/iceberg-rest:0.2.0
+    ports:
+      - ${REST_CATALOG_PORT}:8181
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+      - CATALOG_WAREHOUSE=s3a://warehouse/wh/
+      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+      - CATALOG_S3_ENDPOINT=http://doris--minio:9000
+    networks:
+      - doris--iceberg
+  doris--minio:
+    image: minio/minio
+    container_name: doris--minio
+    hostname: doris--minio
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=password
+    ports:
+      - ${MINIO_UI_PORT}:9001
+      - ${MINIO_API_PORT}:9000
+    networks:
+      - doris--iceberg
+    command: ["server", "/data", "--console-address", ":9001"]
+  doris--mc:
+    depends_on:
+      - doris--minio
+    image: minio/mc
+    container_name: doris--mc
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+    networks:
+      - doris--iceberg
+    entrypoint: >
+      /bin/sh -c "
+      until (/usr/bin/mc config host add minio http://doris--minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc rm -r --force minio/warehouse;
+      /usr/bin/mc mb minio/warehouse;
+      /usr/bin/mc policy set public minio/warehouse;
+      exit 0;
+      "
+networks:
+  doris--iceberg:

docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl

@@ -0,0 +1,11 @@
+spark.sql.catalog.demo                 org.apache.iceberg.spark.SparkCatalog
+spark.sql.catalog.demo.catalog-impl    org.apache.iceberg.rest.RESTCatalog
+spark.sql.catalog.demo.uri             http://doris--rest:8181
+spark.sql.catalog.demo.io-impl         org.apache.iceberg.aws.s3.S3FileIO
+spark.sql.catalog.demo.warehouse       s3a://warehouse/wh/
+spark.sql.catalog.demo.s3.endpoint     http://doris--minio:9000
+spark.sql.defaultCatalog               demo
+spark.eventLog.enabled                 true
+spark.eventLog.dir                     /home/iceberg/spark-events
+spark.history.fs.logDirectory          /home/iceberg/spark-events
+spark.sql.catalogImplementation        in-memory

docker/thirdparties/docker-compose/oracle/oracle-11.yaml → docker/thirdparties/docker-compose/oracle/oracle-11.yaml.tpl

@@ -46,4 +46,4 @@ services:
         condition: service_healthy 
 
 networks:
-  doris--oracle_11:
+  doris--oracle_11: