Transport Layer Security (TLS) (#15)

.gitignore

@@ -15,3 +15,5 @@ __pycache__/
 # MySQL login files
 **/.mylogin.cnf
 **/mysql_root_password.txt
+**/superset_cluster_ca_certificate.pem
+**/superset_cluster_ca_key.pem

README.md

@@ -4,7 +4,7 @@
 
 # superset-cluster
 
-Apache Superset against MySQL InnoDB cluster.
+Resilent Business Intelligence.
 [That's how it works](docs/ARCHITECTURE.md).
 
 ## Requirements
@@ -17,14 +17,15 @@ _[See how to do it with `systemctl`](https://documentation.suse.com/smart/system
 * Nodes must be able to resolve DNS names between each other.
 * The user's host must be able to `ssh` to each of the nodes passwordlessly.
 * There should be at least one available and running network interface capable of sending and receiving packets between the user's host and management nodes via IPv4.
-* Ability to read/write to the `/opt` directory on the nodes.
+* Ability to read/write to the `/opt` directory on the nodes as well as in the user's host.
 * On the MySQL nodes port `3306` should be open for communication within the nodes.
 * On the Management nodes port `6446` should be open for communication within the nodes.
+* ... `2377` port for swarm cluster ...
 * For production setups follow [SECURITY.md](docs/SECURITY.md).
 
 ### Installed software
 
-The following software needs to be installed on both the user's host and external nodes. The setup has been tested on [`ubuntu:22.04`](tests/setup/Dockerfile) with the following versions:
+The following software needs to be installed on both the user's host and external nodes. The setup has been tested on [`ubuntu:24.04`](tests/setup/Dockerfile) with the following versions:
 
 * `ca-certificates v20230311ubuntu0.22.04.1`
 * `containerd.io v1.6.31-1`
@@ -34,6 +35,8 @@ The following software needs to be installed on both the user's host and externa
 * `docker-ce-cli v5:26.1.0-1~ubuntu.22.04~jammy`
 * `openssh-server v1:8.9p1-3ubuntu0.10`
 
+...python packages... docker
+
 ## Installation & Usage
 
 With the [Requirements](#requirements) satisfied, you can build and run the entire setup from the repository root catalog:
@@ -76,6 +79,10 @@ For development purposes, you can set up and run end-to-end tests from the test
 
 If you notice anything missing, spot a bug, or have an enhancement proposal, feel free to open an issue with the appropriate label. Pull requests are welcome. Please ensure that the tests are updated as necessary.
 
+## Personal contact information
+
+In case of any inquiries, please write to email: [email protected]
+
 ## Additional resources
 
 * [What is Apache Superset?](https://superset.apache.org/docs/intro)

run.sh

@@ -1,26 +1,17 @@
 #!/bin/bash
-# 10.145.211.156 10.145.211.158
-# 10.145.211.152 10.145.211.153 10.145.211.154
-mgmt_nodes=("wiktor-min-sles" "wiktor-min-rhlike")
-mysql_nodes=("wiktor-ctl" "wiktor-srv" "wiktor-pxy")
-superset_network_interface="tun0"
 
-virtual_ip_address="10.145.211.155"
-virtual_network_interface="eth0"
+mgmt_nodes=("wiktor-min-deblike" "wiktor-min-rhlike")
+mysql_nodes=("wiktor-min-build" "wiktor-cli-sles" "wiktor-minssh-sles")
+superset_network_interface="enp1s0"
+
+virtual_ip_address="10.145.211.180"
+virtual_network_interface="ens3"
+virtual_ip_address_mask="22"
+
+VIRTUAL_NETWORK="10.145.208.0/22"  # do it in common via python
 
 _path_to_root_catalog="."
 
 source ${_path_to_root_catalog}/src/common.sh
 
-start_superset() {
-  docker network create --driver overlay --attachable superset-network
-  echo $(openssl rand -base64 42) | docker secret create superset_secret_key -
-  ./services/redis/init.sh
-  ./services/superset/init.sh ${virtual_ip_address}
-}
-
 initialize_nodes
-superset_node_address=$(get_superset_node_ip ${superset_network_interface})
-docker_swarm_token=$(init_and_get_docker_swarm_token ${superset_node_address})
-clusterize_nodes
-start_superset

services/mysql-mgmt/Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:22.04
 
-COPY . "/opt"
+COPY --chown=superset:superset . "/opt"
 
 RUN \
   apt \
@@ -14,6 +14,7 @@ RUN \
       xz-utils \
       keepalived \
       mysql-client \
+      ifmetric \
       sudo \
   && \
   groupadd \

services/mysql-mgmt/docker-compose.yml

@@ -4,38 +4,75 @@ services:
     build: "."
     image: "mysql-mgmt"
     user: "root"
-    entrypoint: [ "/opt/entrypoint.sh" ]
     container_name: "mysql-mgmt-initcontainer"
     network_mode: "host"
     restart: "no"
-    environment:
-      IS_PRIMARY_MGMT_NODE: "${IS_PRIMARY_MGMT_NODE}"
-      VIRTUAL_IP_ADDRESS: "${VIRTUAL_IP_ADDRESS}"
-      VIRTUAL_NETWORK_INTERFACE: "${VIRTUAL_NETWORK_INTERFACE}"
-      PRIMARY_MYSQL_NODE: "${PRIMARY_MYSQL_NODE}"
-      SECONDARY_FIRST_MYSQL_NODE: "${SECONDARY_FIRST_MYSQL_NODE}"
-      SECONDARY_SECOND_MYSQL_NODE: "${SECONDARY_SECOND_MYSQL_NODE}"
-      MYSQL_TEST_LOGIN_FILE: "/opt/.mylogin.cnf"
+    entrypoint:
+      - "/bin/bash"
+      - "-c"
+      - |
+        set -euo pipefail
+        export VIRTUAL_IP_ADDRESS="${VIRTUAL_IP_ADDRESS}"
+        export VIRTUAL_IP_ADDRESS_MASK="${VIRTUAL_IP_ADDRESS_MASK}"
+        export VIRTUAL_NETWORK_INTERFACE="${VIRTUAL_NETWORK_INTERFACE}"
+        export VIRTUAL_NETWORK="${VIRTUAL_NETWORK}"
+        export MYSQL_TEST_LOGIN_FILE="/opt/.mylogin.cnf"
+        mv "/opt/superset_cluster_ca_certificate.pem" "/opt/default/superset_cluster_ca_certificate.pem"
+        mv "/opt/mysql_router_${MGMT_NODE_TYPE}_certificate.pem" "/opt/default/mysql_router_${MGMT_NODE_TYPE}_certificate.pem"
+        mv "/opt/mysql_router_${MGMT_NODE_TYPE}_key.pem" "/opt/default/mysql_router_${MGMT_NODE_TYPE}_key.pem"
+        if [ "${MGMT_NODE_TYPE}" = "primary" ]; then
+          export STATE="MASTER"
+          export PRIORITY="100"
+          export NODE_IP_ADDRESS=${VIRTUAL_IP_ADDRESS}
+          mysqlsh --login-path="${PRIMARY_MYSQL_NODE}" --execute="dba.configureInstance('${PRIMARY_MYSQL_NODE}')"
+          mysqlsh --login-path="${SECONDARY_FIRST_MYSQL_NODE}" --execute="dba.configureInstance('${SECONDARY_FIRST_MYSQL_NODE}')"
+          mysqlsh --login-path="${SECONDARY_SECOND_MYSQL_NODE}" --execute="dba.configureInstance('${SECONDARY_SECOND_MYSQL_NODE}')"
+          mysqlsh --login-path="${PRIMARY_MYSQL_NODE}" --execute="dba.createCluster('superset');"
+          mysqlsh --login-path="${SECONDARY_FIRST_MYSQL_NODE}" --sql --execute="RESET MASTER;"
+          mysqlsh --login-path="${PRIMARY_MYSQL_NODE}" --execute="dba.getCluster('superset').addInstance('${SECONDARY_FIRST_MYSQL_NODE}',{recoveryMethod:'incremental'});"
+          mysqlsh --login-path="${SECONDARY_SECOND_MYSQL_NODE}" --sql --execute="RESET MASTER;"
+          mysqlsh --login-path="${PRIMARY_MYSQL_NODE}" --execute="dba.getCluster('superset').addInstance('${SECONDARY_SECOND_MYSQL_NODE}',{recoveryMethod:'incremental'});"
+          /opt/envsubst-Linux-x86_64 < "/opt/superset-user.sql.tpl" > "/opt/superset-user.sql"
+          mysqlsh --login-path="${PRIMARY_MYSQL_NODE}" --sql --file="/opt/superset-user.sql"
+        else
+          export STATE="BACKUP"
+          export PRIORITY="90"
+          export NODE_IP_ADDRESS=$(ip -oneline -4 addr show ${VIRTUAL_NETWORK_INTERFACE} | awk '{print $4}' | cut --delimiter=/ --fields=1 | head --lines=1)
+          /opt/envsubst-Linux-x86_64 < "/opt/superset-user.sql.tpl" > "/opt/superset-user.sql"
+          mysqlsh --login-path="${PRIMARY_MYSQL_NODE}" --sql --file="/opt/superset-user.sql"
+          mysqlrouter --user "superset" --bootstrap "superset:cluster@${PRIMARY_MYSQL_NODE}:3306" --directory "/opt/default/mysql_router" --conf-use-sockets --client-ssl-mode="REQUIRED" --server-ssl-mode="REQUIRED" --ssl-ca="/opt/default/superset_cluster_ca_certificate.pem" --client-ssl-cert="/opt/default/mysql_router_${MGMT_NODE_TYPE}_certificate.pem" --client-ssl-key="/opt/default/mysql_router_${MGMT_NODE_TYPE}_key.pem"
+          mysqlsh --login-path="${PRIMARY_MYSQL_NODE}" --sql --execute="DROP USER 'superset'@'$(ip -oneline -4 addr show ${VIRTUAL_NETWORK_INTERFACE} | awk '{print $4}' | cut --delimiter=/ --fields=1 | head --lines=1)';"
+        fi
+        ifmetric ${VIRTUAL_NETWORK_INTERFACE} 2
+        /opt/envsubst-Linux-x86_64 < "/opt/keepalived.conf.tpl" > "/opt/default/keepalived.conf";
+        chown --recursive superset:superset "/opt/default"
+    cap_add:
+      - "NET_ADMIN"
     volumes:
-      - "initcontainer_generated:/opt/initcontainer"
+      - "default_generated:/opt/default"
 
   maincontainer:
     image: "mysql-mgmt"
     user: "superset"
     entrypoint:
-      - "sh"
+      - "/bin/bash"
       - "-c"
       - |
-        sudo keepalived --use-file /opt/initcontainer/keepalived.conf \
-        && mysqlrouter --config /opt/initcontainer/mysql_router/mysqlrouter.conf
+        set -euo pipefail
+        sudo keepalived --use-file "/opt/default/keepalived.conf"
+        sleep ${HEALTHCHECK_START_PERIOD}
+        if [ ! -d "/opt/default/mysql_router/" ]; then
+          mysqlrouter --user "superset" --bootstrap "superset:cluster@${PRIMARY_MYSQL_NODE}:3306" --directory "/opt/default/mysql_router" --conf-use-sockets --client-ssl-mode="REQUIRED" --server-ssl-mode="REQUIRED" --ssl-ca="/opt/default/superset_cluster_ca_certificate.pem" --client-ssl-cert="/opt/default/mysql_router_${MGMT_NODE_TYPE}_certificate.pem" --client-ssl-key="/opt/default/mysql_router_${MGMT_NODE_TYPE}_key.pem"
+        fi
+        mysqlrouter --config "/opt/default/mysql_router/mysqlrouter.conf"
     container_name: "mysql-mgmt"
     restart: "always"
     hostname: "${HOSTNAME}"
     network_mode: "host"
     cap_add:
       - "NET_ADMIN"
     volumes:
-      - "initcontainer_generated:/opt/initcontainer"
+      - "default_generated:/opt/default"
 
 volumes:
-  initcontainer_generated:
+  default_generated:

services/mysql-mgmt/init.sh

@@ -1,11 +1,46 @@
 #!/bin/bash
 
-export IS_PRIMARY_MGMT_NODE="${1}"
+export MGMT_NODE_TYPE="${1}"
 export VIRTUAL_IP_ADDRESS="${2}"
-export VIRTUAL_NETWORK_INTERFACE="${3}"
-export PRIMARY_MYSQL_NODE="${4}"
-export SECONDARY_FIRST_MYSQL_NODE="${5}"
-export SECONDARY_SECOND_MYSQL_NODE="${6}"
+export VIRTUAL_IP_ADDRESS_MASK="${3}"
+export VIRTUAL_NETWORK_INTERFACE="${4}"
+export VIRTUAL_NETWORK="${5}"
+export PRIMARY_MYSQL_NODE="${6}"
+export SECONDARY_FIRST_MYSQL_NODE="${7}"
+export SECONDARY_SECOND_MYSQL_NODE="${8}"
+export HEALTHCHECK_START_PERIOD=25  # must be greater than vrrp_startup_delay
+export HEALTHCHECK_INTERVAL=5
+export HEALTHCHECK_RETRIES=3
+
+openssl \
+  genpkey \
+    -algorithm RSA \
+    -out "/opt/superset-cluster/mysql-mgmt/mysql_router_${MGMT_NODE_TYPE}_key.pem"
+
+openssl \
+  req \
+    -new \
+    -key "/opt/superset-cluster/mysql-mgmt/mysql_router_${MGMT_NODE_TYPE}_key.pem" \
+    -out "/opt/superset-cluster/mysql-mgmt/mysql_router_${MGMT_NODE_TYPE}_certificate_signing_request.pem" \
+    -subj "/CN=Superset-Cluster-MySQL-Router-${MGMT_NODE_TYPE}"
+
+openssl \
+  x509 \
+    -in "/opt/superset-cluster/mysql-mgmt/mysql_router_${MGMT_NODE_TYPE}_certificate_signing_request.pem" \
+    -CA "/opt/superset-cluster/mysql-mgmt/superset_cluster_ca_certificate.pem" \
+    -CAkey "/opt/superset-cluster/mysql-mgmt/superset_cluster_ca_key.pem" \
+    -CAcreateserial \
+    -out "/opt/superset-cluster/mysql-mgmt/mysql_router_${MGMT_NODE_TYPE}_certificate.pem" \
+    -req \
+    -days 365
+
+docker run \
+  --detach \
+  --restart always \
+  --name redis \
+  --hostname redis \
+  --network superset-network \
+  redis
 
 docker compose \
   --file /opt/superset-cluster/mysql-mgmt/docker-compose.yml up initcontainer \

services/mysql-mgmt/keepalived.conf.tpl

@@ -1,3 +1,8 @@
+
+global_defs {
+    vrrp_startup_delay 15
+}
+
 vrrp_script status {
     script "/bin/killall -0 mysqlrouter"
     interval 2
@@ -10,10 +15,14 @@ vrrp_instance virtual_instance {
     virtual_router_id 51
     priority ${PRIORITY}
     advert_int 1
+    nopreempt
     track_script {
         status
     }
     virtual_ipaddress {
-        ${VIRTUAL_IP_ADDRESS}
+        ${VIRTUAL_IP_ADDRESS}/${VIRTUAL_IP_ADDRESS_MASK}
+    }
+    virtual_routes {
+        ${VIRTUAL_NETWORK} src ${VIRTUAL_IP_ADDRESS} metric 1 dev ${VIRTUAL_NETWORK_INTERFACE} scope link
     }
 }

services/mysql-mgmt/superset-user.sql.tpl

@@ -0,0 +1,10 @@
+CREATE USER IF NOT EXISTS 'superset'@'${NODE_IP_ADDRESS}' IDENTIFIED BY 'cluster';
+CREATE DATABASE IF NOT EXISTS `superset`;
+GRANT ALL PRIVILEGES ON `superset`.* TO 'superset'@'${NODE_IP_ADDRESS}';
+GRANT INSERT ON `mysql_innodb_cluster_metadata`.* TO 'superset'@'${NODE_IP_ADDRESS}';
+GRANT SELECT ON `performance_schema`.* TO 'superset'@'${NODE_IP_ADDRESS}' WITH GRANT OPTION;
+GRANT CREATE USER ON *.* TO 'superset'@'${NODE_IP_ADDRESS}';
+GRANT SELECT, EXECUTE ON `mysql_innodb_cluster_metadata`.* TO 'superset'@'${NODE_IP_ADDRESS}' WITH GRANT OPTION;
+GRANT INSERT, UPDATE, DELETE ON `mysql_innodb_cluster_metadata`.`routers` TO 'superset'@'${NODE_IP_ADDRESS}' WITH GRANT OPTION;
+GRANT INSERT, UPDATE, DELETE ON `mysql_innodb_cluster_metadata`.`v2_routers` TO 'superset'@'${NODE_IP_ADDRESS}' WITH GRANT OPTION;
+GRANT SELECT ON `mysql`.`user` TO 'superset'@'${NODE_IP_ADDRESS}';

services/mysql-server/Dockerfile

@@ -3,7 +3,16 @@ FROM mysql:8.0-debian
 ARG SERVER_ID
 ENV SERVER_ID="${SERVER_ID}"
 
-COPY "mysql_root_password.txt" "mysql_config.cnf.tpl" "/opt/"
+COPY \
+  "mysql_root_password.txt" \
+  "mysql_config.cnf.tpl" \
+    "/opt/"
+
+COPY \
+  "mysql_server_${SERVER_ID}_certificate.pem" \
+  "mysql_server_${SERVER_ID}_key.pem" \
+  "superset_cluster_ca_certificate.pem" \
+    "/etc/mysql/ssl/"
 
 RUN \
   apt \
@@ -24,12 +33,16 @@ RUN \
     500 \
       envsubst \
   && \
-    ./envsubst < "/opt/mysql_config.cnf.tpl" > "/etc/mysql/conf.d/mysql_config.cnf" \
+    ./envsubst \
+      -no-unset \
+      -no-empty \
+        < "/opt/mysql_config.cnf.tpl" > "/etc/mysql/conf.d/mysql_config.cnf" \
   && \
   chown \
     --recursive \
       mysql:mysql \
         "/opt" \
+        "/etc/mysql/ssl" \
   && \
   rm \
     --recursive \