new idea with doing exec on root user after container start, conside …

…seccomp & selinux & apparmor

services/mysql-mgmt/entrypoint.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-if [ "${IS_PRIMARY_MGMT_NODE}" == "true" ]; then
+if [ "${IS_PRIMARY_MGMT_NODE}" = "true" ]; then
   export STATE="MASTER"
   export PRIORITY="100"
 
@@ -27,7 +27,7 @@ mysqlrouter --user "superset" --bootstrap "superset:cluster@${PRIMARY_MYSQL_NODE
 /opt/envsubst-Linux-x86_64 < "/opt/keepalived.conf.tpl" > "/opt/initcontainer/keepalived.conf"
 chown "superset:superset" "/opt/initcontainer/keepalived.conf"
 
-if [ "${ENVIRONMENT}" == "testing" ]; then
+if [ "${ENVIRONMENT}" = "testing" ]; then
   mv "/opt/.mylogin.cnf" "/opt/initcontainer/.mylogin.cnf"
   chown "superset:superset" "/opt/initcontainer/.mylogin.cnf"
 fi

services/mysql-mgmt/init.sh

@@ -17,13 +17,13 @@ docker compose up initcontainer && docker compose up maincontainer --detach
 # ip addr add 192.168.1.100/24 dev eth0
 
 
-# export ENVIRONMENT=testing
-# export IS_PRIMARY_MGMT_NODE=true
-# export VIRTUAL_IP_ADDRESS=172.18.0.8
-# export VIRTUAL_NETWORK_INTERFACE=eth0
-# export PRIMARY_MYSQL_NODE=node-1
-# export SECONDARY_FIRST_MYSQL_NODE=node-2
-# export SECONDARY_SECOND_MYSQL_NODE=node-3
+export ENVIRONMENT=testing
+export IS_PRIMARY_MGMT_NODE=true
+export VIRTUAL_IP_ADDRESS=172.18.0.8
+export VIRTUAL_NETWORK_INTERFACE=eth0
+export PRIMARY_MYSQL_NODE=node-1
+export SECONDARY_FIRST_MYSQL_NODE=node-2
+export SECONDARY_SECOND_MYSQL_NODE=node-3
 
 # cd /opt/superset-cluster/mysql-mgmt
 # docker compose up initcontainer -d

services/mysql-server/Dockerfile

@@ -3,7 +3,7 @@ FROM mysql:8.0-debian
 ARG SERVER_ID
 ENV SERVER_ID="${SERVER_ID}"
 
-COPY . "/opt"
+COPY "mysql_root_password.txt" "mysql_config.cnf.tpl" "/opt/"
 
 RUN \
   apt \
@@ -13,7 +13,6 @@ RUN \
     install \
       --yes \
       curl \
-      expect \
   && \
   curl \
     --location \
@@ -27,54 +26,14 @@ RUN \
   && \
     ./envsubst < "/opt/mysql_config.cnf.tpl" > "/etc/mysql/conf.d/mysql_config.cnf" \
   && \
-  groupadd \
-    --system \
-      "superset" \
-  && \
-  useradd \
-    --system \
-    --gid \
-      "superset" \
-    --create-home \
-    --home-dir \
-      "/home/superset" \
-    --shell \
-      "/bin/bash" \
-    superset \
-  && \
-  mv \
-    "/opt/entrypoint.sh" \
-    "/opt/store_credentials" \
-    "/home/superset" \
-  && \
-  if [ -f "/opt/.mylogin.cnf" ]; then \
-    mv "/opt/.mylogin.cnf" "/home/superset/"; \
-  fi \
-  && \
   chown \
     --recursive \
-      superset:superset \
-        "/var/lib/mysql" \
-        "/var/run/mysqld" \
-        "/home/superset" \
-  && \
-  chmod \
-    500 \
-      "/home/superset/entrypoint.sh" \
-  && \
-  chmod \
-    500 \
-      "/home/superset/store_credentials" \
+      mysql:mysql \
+        "/opt" \
   && \
   rm \
     --recursive \
     --force \
       "/var/lib/apt/lists/*"
 
-USER superset
-
-WORKDIR "/home/superset"
-
-ENV MYSQL_TEST_LOGIN_FILE="/home/superset/.mylogin.cnf"
-
-ENTRYPOINT [ "/home/superset/entrypoint.sh" ]
+USER mysql

services/mysql-server/default.json

@@ -0,0 +1,15 @@
+{
+  "defaultAction": "SCMP_ACT_ERRNO",
+  "architectures": ["SCMP_ARCH_X86_64"],
+  "syscalls": [
+    {
+      "names": [
+        "kill",
+        "pkill",
+        "killall"
+      ],
+      "action": "SCMP_ACT_ERRNO"
+    }
+  ]
+}
+

services/mysql-server/init.sh

@@ -11,5 +11,10 @@ docker run \
   --name mysql \
   --hostname "${HOSTNAME}" \
   --network host \
-  --cap-add sys_nice \
+  --cap-add SYS_NICE \
+  --security-opt seccomp=default.json \
+  --env MYSQL_INITDB_SKIP_TZINFO="true" \
+  --env MYSQL_ROOT_PASSWORD_FILE="/opt/mysql_root_password.txt" \
   mysql-server
+
+docker exec --user=root mysql /bin/bash -c "chmod 400 /opt/mysql_root_password.txt && chown --recursive root:root /opt /var/run/mysqld"

services/mysql-server/mysql_config.cnf.tpl

@@ -1,9 +1,11 @@
 [mysqld]
-server_id="${SERVER_ID}"
-disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY"
-performance_schema="ON"
-transaction_isolation="READ-COMMITTED"
-binlog_transaction_dependency_tracking="WRITESET"
-enforce_gtid_consistency="ON"
-gtid_mode="ON"
-pid-file="/home/superset/mysqld.pid"
+user = "mysql"
+server_id = "${SERVER_ID}"
+disabled_storage_engines = "MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY"
+performance_schema = "ON"
+transaction_isolation = "READ-COMMITTED"
+binlog_transaction_dependency_tracking = "WRITESET"
+binlog_encryption = "ON"
+enforce_gtid_consistency = "ON"
+gtid_mode = "ON"
+pid-file = "/var/run/mysqld/mysqld.pid"

services/mysql-server/mysql_root_password.txt

@@ -0,0 +1 @@
+c3ed1b5822112464e8

services/superset/Dockerfile

@@ -35,15 +35,13 @@ RUN \
     db \
       upgrade \
   && \
-  superset load_examples \
+  if [ "${ENVIRONMENT}" = "testing" ]; then \
+    superset load_examples; \
+  fi \
   &&  \
   superset \
     init
 
 ENTRYPOINT [ "sh", "-c", \
   "celery --app superset.tasks.celery_app:app worker --pool prefork --concurrency 4 -O fair --detach && /usr/bin/run-server.sh" \
 ]
-
-# if [ "${ENVIRONMENT}" == "testing" ]; then \
-#     superset load_examples; \
-#   fi \

src/common.sh

@@ -9,21 +9,24 @@ array_to_string_converter() {
 }
 
 initialize_nodes() {
+  export MYSQL_TEST_LOGIN_FILE="${_path_to_root_catalog}/services/mysql-mgmt/.mylogin.cnf"
+  ./${_path_to_root_catalog}/src/store_credentials node-1 node-2 node-3 ${_path_to_root_catalog}
   for mysql_node in "${mysql_nodes[@]}"; do
     ssh root@${mysql_node} "mkdir --parents /opt/superset-cluster"
     scp -r "${_path_to_root_catalog}/services/mysql-server" "root@${mysql_node}:/opt/superset-cluster"
-    ssh root@${mysql_node} "/opt/superset-cluster/mysql-server/init.sh"
-    ssh root@${mysql_node} "docker cp mysql:/home/superset/.mylogin.cnf /opt/superset-cluster/mysql-server/"
-    scp "root@${mysql_node}:/opt/superset-cluster/mysql-server/.mylogin.cnf" "${_path_to_root_catalog}/services/mysql-server/"
-  done
-  mv "${_path_to_root_catalog}/services/mysql-server/.mylogin.cnf" "${_path_to_root_catalog}/services/mysql-mgmt/"
-  IS_PRIMARY_MGMT_NODE=true
-  for mgmt_node in "${mgmt_nodes[@]}"; do
-    ssh root@${mgmt_node} "mkdir --parents /opt/superset-cluster"
-    scp -r ${_path_to_root_catalog}/services/mysql-mgmt "root@${mgmt_node}:/opt/superset-cluster"
-    ssh root@${mgmt_node} "/opt/superset-cluster/mysql-mgmt/init.sh ${ENVIRONMENT} ${IS_PRIMARY_MGMT_NODE} ${virtual_ip_address} ${virtual_network_interface} $(array_to_string_converter ${mysql_nodes[@]})"
-    IS_PRIMARY_MGMT_NODE=false
+    # ssh root@${mysql_node} "/opt/superset-cluster/mysql-server/init.sh"
+    # ssh root@${mysql_node} "rm /opt/superset-cluster/mysql-server/mysql_root_password.txt"
+    # ssh root@${mysql_node} "docker cp mysql:/opt/.mylogin.cnf /opt/superset-cluster/mysql-server/"
+    # scp "root@${mysql_node}:/opt/superset-cluster/mysql-server/.mylogin.cnf" "${_path_to_root_catalog}/services/mysql-server/"
   done
+  # # mv "${_path_to_root_catalog}/services/mysql-server/.mylogin.cnf" "${_path_to_root_catalog}/services/mysql-mgmt/"
+  # IS_PRIMARY_MGMT_NODE=true
+  # for mgmt_node in "${mgmt_nodes[@]}"; do
+  #   ssh root@${mgmt_node} "mkdir --parents /opt/superset-cluster"
+  #   scp -r ${_path_to_root_catalog}/services/mysql-mgmt "root@${mgmt_node}:/opt/superset-cluster"
+  #   ssh root@${mgmt_node} "/opt/superset-cluster/mysql-mgmt/init.sh ${ENVIRONMENT} ${IS_PRIMARY_MGMT_NODE} ${virtual_ip_address} ${virtual_network_interface} $(array_to_string_converter ${mysql_nodes[@]})"
+  #   IS_PRIMARY_MGMT_NODE=false
+  # done
 }
 
 get_superset_node_ip() {

src/store_credentials

@@ -0,0 +1,44 @@
+#!/usr/bin/expect -f
+
+set node1 [lindex $argv 0]
+set node2 [lindex $argv 1]
+set node3 [lindex $argv 2]
+set path_to_root_catalog [lindex $argv 3]
+set password [exec openssl rand -hex 9]
+set password_file "$path_to_root_catalog/services/mysql-server/mysql_root_password.txt"
+
+set fp [open $password_file w]
+puts $fp $password
+close $fp
+
+spawn mysql_config_editor set \
+  --login-path=$node1 \
+  --host=$node1 \
+  --user=root \
+  --skip-warn \
+  --password
+
+expect "Enter password:"
+send "$password\r"
+
+spawn mysql_config_editor set \
+  --login-path=$node2 \
+  --host=$node2 \
+  --user=root \
+  --skip-warn \
+  --password
+
+expect "Enter password:"
+send "$password\r"
+
+spawn mysql_config_editor set \
+  --login-path=$node3 \
+  --host=$node3 \
+  --user=root \
+  --skip-warn \
+  --password
+
+expect "Enter password:"
+send "$password\r"
+
+expect eof

tests/testsuite/deploy.yml

@@ -13,9 +13,9 @@
   tasks:
     - ansible.builtin.include_role: { name: "testing", tasks_from: "system" }
 
-- name: "functional testing"
-  connection: "local"
-  hosts: "testing"
-  any_errors_fatal: yes
-  tasks:
-    - ansible.builtin.include_role: { name: "testing", tasks_from: "functional" }
+# - name: "functional testing"
+#   connection: "local"
+#   hosts: "testing"
+#   any_errors_fatal: yes
+#   tasks:
+#     - ansible.builtin.include_role: { name: "testing", tasks_from: "functional" }

tests/testsuite/roles/testing/tasks/functional.yml

@@ -44,6 +44,10 @@
     name: "{{ node_prefix }}-1"
     state: stopped
 
+- name: Pause for 60 seconds
+  pause:
+    seconds: 60
+
 - name: "run mgmt functional tests after primary nodes disaster"
   community.docker.docker_container_exec:
     container: "{{ node_prefix }}-5"

tests/testsuite/roles/testing/tasks/system.yml

@@ -8,10 +8,5 @@
       {{ virtual_ip_address }} \
       {{ virtual_network_interface }}
     initialize_nodes
-    superset_node_address=$(superset_node_address)
-    docker_swarm_token=$(docker_swarm_token "${superset_node_address}")
-    clusterize_nodes
-    start_superset
-    sleep 60
   args:
     executable: /bin/bash