[Enhancement] submit Baidu test cases (apache#22883)

.gitignore

@@ -20,6 +20,7 @@ dependency-reduced-pom.xml
 yarn.lock
 ut_dir
 package-lock.json
+__pycache__
 
 # ignore all sub `/bin` and `/build` directory
 **/build/

.licenserc.yaml

@@ -88,5 +88,8 @@ header:
     - "regression-test/ssl_default_certificate/*"
     - "extension/beats/go.mod"
     - "extension/beats/go.sum"
-
+    - "pytest/hdfs"
+    - "pytest/qe"
+    - "pytest/sys/data"
+    - "pytest/deploy/*.conf"
   comment: on-failure

pytest/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

pytest/deploy/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

pytest/deploy/backup.py

@@ -0,0 +1,106 @@
+#!/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+This module init Palo runtime environment.
+Date:    2015/10/07 17:23:06
+"""
+import datetime
+import sys
+import threading
+
+import execute
+import env_config
+import stop
+
+fmt = '%Y-%m-%d-%H-%M-%S-%f'
+
+
+def dump_fe(host_name):
+    """dump fe
+    """
+    cmd = 'cd %s/fe;jmap -dump:live,format=b,file=heap.bin `cat bin/fe.pid`' \
+            % env_config.fe_path
+    status, output = execute.exe_cmd(cmd, host_name)
+
+
+def backup_one_fe(host_name, backup_name):
+    """backup one fe
+    """
+    dump_fe(host_name)
+    stop.stop_one_fe(host_name)
+    cmd = 'cd {0};mkdir -p backup/{1};mv {2} backup/{1}'.format(
+            env_config.palo_path, backup_name, 'PALO-FE')
+    status, output = execute.exe_cmd(cmd, host_name)
+
+
+def backup_fe(backup_name=None):
+    """backup fe
+    """
+    if not backup_name:
+        backup_name = datetime.datetime.now().strftime(fmt)
+    backup_fe_threads = []
+    for host_name in [env_config.master] + env_config.follower_list + \
+            env_config.observer_list + env_config.dynamic_add_fe_list:
+        t = threading.Thread(target=backup_one_fe, args=(host_name, backup_name))
+        t.start()
+        backup_fe_threads.append(t)
+
+    for t in backup_fe_threads:
+        t.join()
+
+
+def backup_one_be(host_name, backup_name):
+    """backup one be
+    """
+    stop.stop_one_be(host_name)
+    cmd = 'cd {0};mkdir -p backup/{1};mv {2} backup/{1}'.format(
+            env_config.palo_path, backup_name, 'PALO-BE')
+    status, output = execute.exe_cmd(cmd, host_name)
+
+
+def backup_be(backup_name=None):
+    """backup be
+    """
+    if not backup_name:
+        backup_name = datetime.datetime.now().strftime(fmt)
+    backup_be_threads = []
+    for host_name in env_config.be_list + env_config.dynamic_add_be_list:
+        t = threading.Thread(target=backup_one_be, args=(host_name, backup_name))
+        t.start()
+        backup_be_threads.append(t)
+
+    for t in backup_be_threads:
+        t.join()
+
+
+def backup_palo(backup_name=None):
+    """backup palo
+    """
+    if not backup_name:
+        backup_name = datetime.datetime.now().strftime(fmt)
+    backup_fe(backup_name)
+    backup_be(backup_name)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) == 2:
+        backup_palo(sys.argv[1])
+    else:
+        backup_palo()

pytest/deploy/be.conf

@@ -0,0 +1,40 @@
+# INFO, WARNING, ERROR, FATAL
+sys_log_level = INFO
+
+# ports for admin, web, heartbeat service 
+be_port = 
+webserver_port = 
+heartbeat_service_port = 
+be_rpc_port =
+brpc_port = 
+
+# data root path, seperate by ';' 
+storage_root_path = ${DORIS_HOME}/data/data.SSD;${DORIS_HOME}/data/data.HDD
+
+# Advanced configurations
+# sys_log_dir = ${PALO_HOME}/log
+# sys_log_roll_mode = SIZE-MB-1024
+# sys_log_roll_num = 10
+# sys_log_verbose_modules = *
+# palo_cgroups 
+
+# user-defined
+max_unpacked_row_block_size=1048576000
+
+report_olap_table_interval_seconds = 10
+
+delete_delta_expire_time = 0
+be_policy_cumulative_files_number = 3
+be_policy_cumulative_base_ratio = 0.1
+be_policy_start_time = 0
+be_policy_end_time = 24
+be_policy_be_interval_seconds = 0
+
+ce_policy_delta_files_number = 3
+ce_policy_max_delta_file_size = 256
+
+rpc_reactor_threads = 3
+thrift_rpc_timeout_ms = 10000
+pending_data_expire_time_sec = 60
+
+enable_java_support = false

pytest/deploy/clean.py

@@ -0,0 +1,82 @@
+#!/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+This module init Palo runtime environment.
+Date:    2015/10/07 17:23:06
+"""
+
+import threading
+import execute
+import env_config
+import stop
+
+
+def clean_one_fe(host_name):
+    """clean one fe
+    """
+    stop.stop_one_fe(host_name)
+    cmd = 'cd %s;rm PALO-FE -rf' % env_config.palo_path
+    status, output = execute.exe_cmd(cmd, host_name)
+
+
+def clean_fe():
+    """clean fe
+    """
+    clean_fe_threads = []
+    for host_name in [env_config.master] + env_config.follower_list + \
+            env_config.observer_list + env_config.dynamic_add_fe_list:
+        t = threading.Thread(target=clean_one_fe, args=(host_name,))
+        t.start()
+        clean_fe_threads.append(t)
+
+    for t in clean_fe_threads:
+        t.join()
+
+
+def clean_one_be(host_name):
+    """clean one be
+    """
+    stop.stop_one_be(host_name)
+    cmd = 'cd %s;rm PALO-BE -rf' % (env_config.palo_path)
+    status, output = execute.exe_cmd(cmd, host_name)
+
+
+def clean_be():
+    """clean be
+    """
+    clean_be_threads = []
+    for host_name in env_config.be_list + env_config.dynamic_add_be_list:
+        t = threading.Thread(target=clean_one_be, args=(host_name,))
+        t.start()
+        clean_be_threads.append(t)
+
+    for t in clean_be_threads:
+        t.join()
+
+
+def clean_palo():
+    """clean palo
+    """
+    clean_fe()
+    clean_be()
+
+
+if __name__ == '__main__':
+    clean_palo()

pytest/deploy/clean_backup.py

@@ -0,0 +1,80 @@
+#!/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+This module init Palo runtime environment.
+Date:    2015/10/07 17:23:06
+"""
+
+import threading
+
+import execute
+import env_config
+
+
+def clean_one_fe_backup(host_name):
+    """clean one fe
+    """
+    cmd = 'cd %s;rm backup -rf' % env_config.palo_path
+    status, output = execute.exe_cmd(cmd, host_name)
+
+
+def clean_fe_backup():
+    """clean fe
+    """
+    clean_fe_threads = []
+    for host_name in [env_config.master] + env_config.follower_list + \
+            env_config.observer_list + env_config.dynamic_add_fe_list:
+        t = threading.Thread(target=clean_one_fe_backup, args=(host_name,))
+        t.start()
+        clean_fe_threads.append(t)
+
+    for t in clean_fe_threads:
+        t.join()
+
+
+def clean_one_be_backup(host_name):
+    """clean one be
+    """
+    cmd = 'cd %s;rm backup -rf' % env_config.palo_path
+    status, output = execute.exe_cmd(cmd, host_name)
+
+
+def clean_be_backup():
+    """clean be
+    """
+    clean_be_threads = []
+    for host_name in env_config.be_list + env_config.dynamic_add_be_list:
+        t = threading.Thread(target=clean_one_be_backup, args=(host_name,))
+        t.start()
+        clean_be_threads.append(t)
+
+    for t in clean_be_threads:
+        t.join()
+
+
+def clean_palo_backup():
+    """clean palo
+    """
+    clean_fe_backup()
+    clean_be_backup()
+
+
+if __name__ == '__main__':
+    clean_palo_backup()