scripts to check for worker readiness and liveness

bin/check-worker-liveness

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+: "${DATABASE_URL:?Please set DATABASE_URL}"
+: "${HOSTNAME:?Please set HOSTNAME}"
+
+MIN_RUN_AT_FILE=/tmp/.min-run-at.op
+FAIL_COUNT_FILE=/tmp/.liveness-check-fail-count.op
+
+# How many jobs are overdue at least 15 minutes?
+#
+# Filter by not locked jobs because a crashing worker may leave behind
+# a locked, unprocessed job. But this doesn't mean the new worker isn't working.
+#
+# This checks across all workers and can't tell if it's only this worker in particular
+# that doesn't seem to be doing anything.
+UNPROCESSED_COUNT=`psql -d ${DATABASE_URL%%\?*} -c "select count(*) from delayed_jobs where locked_by is null and run_at < (now() - interval '15 minutes');" | tail -n +3 | head -n1 | tr -d ' '`
+
+echo "unprocessed: $UNPROCESSED_COUNT"
+
+if [ "$UNPROCESSED_COUNT" = "0" ]; then
+  echo 0 > $MIN_RUN_AT_FILE # `date -d '0'` will be the beginning of the day which is fine for our purposes
+  echo 0 > $FAIL_COUNT_FILE
+
+  echo "Workers ok"
+
+  exit 0
+else
+  MIN_RUN_AT=`psql -d ${DATABASE_URL%%\?*} -c "select run_at from delayed_jobs where locked_by is null and cron is null and run_at is not null and run_at < (now() - interval '1 minutes') order by run_at asc limit 1;" | tail -n +3 | head -n1`
+  LAST_MIN_RUN_AT=`cat $MIN_RUN_AT_FILE 2>/dev/null || echo 0`
+  NUM_FAILS=`cat $FAIL_COUNT_FILE 2>/dev/null || echo 0`
+
+  if [ "$MIN_RUN_AT" = "(0 rows)" ]; then
+    echo "There are no unprocessed jobs after all."
+
+    exit 0
+  # check if last dangling job has been processed by now
+  elif [ "`date -d "$MIN_RUN_AT" '+%s'`" -gt "`date -d "$LAST_MIN_RUN_AT" '+%s'`" ]; then
+    echo 0 > $FAIL_COUNT_FILE
+
+    echo "Jobs recovering. Workers seem to be ok."
+    exit 0
+  elif [ "$NUM_FAILS" -lt 3 ]; then
+    echo "$((NUM_FAILS + 1))" > $FAIL_COUNT_FILE
+
+    echo "Job still hasn't been locked. Workers may not be ok. We're keeping an eye on it."
+    exit 0
+  else
+    echo "It's dead, Jim."
+
+    exit 1
+  fi
+fi

bin/check-worker-readiness

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if ps aux | grep 'rake jobs:work' | grep -v grep; then
+  echo "background worker running"
+  exit 0
+fi
+  echo "background worker not running"
+  exit 1
+fi