Merge pull request #94 from mohitj1988/PSTRESS-154

PSTRESS-154 - Add RR support in pstress How to use RR: GitHub: https://github.com/rr-debugger/rr Install and Build: https://github.com/rr-debugger/rr/wiki/Building-And-Installing
Percona-QA · Oct 12, 2023 · b35bcad · b35bcad
2 parents 3e0ed9d + 34d4ce1
commit b35bcad
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 6 deletions.
diff --git a/pstress/pstress-run-57.conf b/pstress/pstress-run-57.conf
@@ -185,3 +185,9 @@ GRP_RPL_CLUSTER_RUN=0
 # Default GR configuration file for multi-node pstress runs                    #
 ################################################################################
 GR_CLUSTER_CONFIG=${SCRIPT_PWD}/pstress-cluster-run.cfg
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0
diff --git a/pstress/pstress-run-80.conf b/pstress/pstress-run-80.conf
@@ -189,3 +189,9 @@ GRP_RPL_CLUSTER_RUN=0
 # Default GR configuration file for multi-node pstress runs                    #
 ################################################################################
 GR_CLUSTER_CONFIG=${SCRIPT_PWD}/pstress-cluster-run.cfg
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0
diff --git a/pstress/pstress-run-PXC57.conf b/pstress/pstress-run-PXC57.conf
@@ -204,3 +204,9 @@ PXC_WSREP_PROVIDER_ADD_RANDOM_WSREP_PROVIDER_CONFIG_OPTIONS=0
 # Maximum number of PXC wsrep provider (Galera) configuration options to add   #
 ################################################################################
 PXC_WSREP_PROVIDER_MAX_NR_OF_RND_OPTS_TO_ADD=2
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0
diff --git a/pstress/pstress-run-PXC80.conf b/pstress/pstress-run-PXC80.conf
@@ -216,3 +216,9 @@ PXC_WSREP_PROVIDER_ADD_RANDOM_WSREP_PROVIDER_CONFIG_OPTIONS=0
 # Maximum number of PXC wsrep provider (Galera) configuration options to add   #
 ################################################################################
 PXC_WSREP_PROVIDER_MAX_NR_OF_RND_OPTS_TO_ADD=2
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0
diff --git a/pstress/pstress-run-rocksdb.conf b/pstress/pstress-run-rocksdb.conf
@@ -152,3 +152,9 @@ MYINIT=
 # Extra options to pass to mysqld during server start                          #
 ################################################################################
 MYEXTRA=
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0
diff --git a/pstress/pstress-run.sh b/pstress/pstress-run.sh
@@ -176,6 +176,24 @@ EOF
   fi
 }
 
+# Incase, user starts pstress in RR mode, check if RR is installed on the machine
+if [ $RR_MODE -eq 1 ]; then
+  echoit "Running pstress in RR mode. It is expected that pstress executions will be slower"
+  if [[ ! -e `which rr` ]];then
+    echo "rr package is not installed. Exiting"
+    echo "Install rr: https://github.com/rr-debugger/rr/wiki/Building-And-Installing"
+    exit 1
+  else
+    perf_event_var=$(cat /proc/sys/kernel/perf_event_paranoid)
+    if [ $perf_event_var -ne 1 ]; then
+      echo "rr needs /proc/sys/kernel/perf_event_paranoid <=1, but it is $perf_event_var"
+      echo "Change it to 1, consider running sudo sysctl -w kernel.perf_event_paranoid=1"
+      echo "For more information https://github.com/rr-debugger/rr/wiki/Building-And-Installing"
+      exit 1
+    fi
+  fi
+fi
+
 # Find mysqld binary
 if [ -r ${BASEDIR}/bin/mysqld ]; then
   BIN=${BASEDIR}/bin/mysqld
@@ -480,15 +498,27 @@ pxc_startup(){
   sed -i "2i wsrep_cluster_address=gcomm://${PXC_LADDRS[1]},${PXC_LADDRS[2]},${PXC_LADDRS[3]}" ${DATADIR}/n3.cnf
 
   get_error_socket_file 1
-  ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n1.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA --wsrep-new-cluster > ${ERR_FILE} 2>&1 &
+  if [ $RR_MODE -eq 1 ]; then
+    rr ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n1.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA --wsrep-new-cluster > ${ERR_FILE} 2>&1 &
+  elif [ $RR_MODE -eq 0 ]; then
+    ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n1.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA --wsrep-new-cluster > ${ERR_FILE} 2>&1 &
+  fi
   pxc_startup_status 1
 
   get_error_socket_file 2
-  ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n2.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  if [ $RR_MODE -eq 1 ]; then
+    rr ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n2.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  elif [ $RR_MODE -eq 0 ]; then
+    ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n2.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  fi
   pxc_startup_status 2
 
   get_error_socket_file 3
-  ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n3.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  if [ $RR_MODE -eq 1 ]; then
+    rr ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n3.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  elif [ $RR_MODE -eq 0 ]; then
+    ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n3.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  fi
   pxc_startup_status 3
 
   if [ "$IS_STARTUP" == "startup" ]; then
@@ -905,6 +935,9 @@ pstress_test(){
         --log-output=none --log-error-verbosity=3 --log-error=${RUNDIR}/${TRIAL}/log/master.err"
     fi
 
+    if [ $RR_MODE -eq 1 ]; then
+      CMD="rr $CMD"
+    fi
     echo $CMD
     $CMD > ${RUNDIR}/${TRIAL}/log/master.err 2>&1 &
     MPID="$!"
@@ -1481,19 +1514,21 @@ elif [[ ${PXC} -eq 1 || ${GRP_RPL} -eq 1 ]]; then
     if ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/node1.template/node1_socket.sock  ping > /dev/null 2>&1; then
       echoit "PXC node1.template started" ;
     else
-      echoit "Assert: PXC data template creation failed.."
+      echoit "Assert: PXC data template1 creation failed.."
       exit 1
     fi
+    sleep 2
     if ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/node2.template/node2_socket.sock  ping > /dev/null 2>&1; then
       echoit "PXC node2.template started" ;
     else
-      echoit "Assert: PXC data template creation failed.."
+      echoit "Assert: PXC data template2 creation failed.."
       exit 1
     fi
+    sleep 2
     if ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/node3.template/node3_socket.sock  ping > /dev/null 2>&1; then
       echoit "PXC node3.template started" ;
     else
-      echoit "Assert: PXC data template creation failed.."
+      echoit "Assert: PXC data template3 creation failed.."
       exit 1
     fi
     echoit "Created PXC data templates for pstress run.."