From e8db18db1b295c8eb382c4d75e95e05a9fe7208a Mon Sep 17 00:00:00 2001
From: Bill Buchan <bill.buchan@digital.justice.gov.uk>
Date: Wed, 8 May 2024 14:26:42 +0100
Subject: [PATCH 1/2] Add New Metric Extensions for Alfresco Messaging

---
 .../collection/ME#24#ALFRESCO_QUEUE_LAG.xml   |  9 +++++++
 .../delius/alfresco_queue_lag/mea.xml         |  6 +++++
 .../metadata/ME#24#ALFRESCO_QUEUE_LAG.xml     | 26 +++++++++++++++++++
 .../scripts/alfresco_queue_lag.sh             | 21 +++++++++++++++
 .../collection/ME#24#FAILED_PDT_THREAD.xml}   |  4 +--
 .../delius/failed_pdt_thread/mea.xml          |  6 +++++
 .../metadata/ME#24#FAILED_PDT_THREAD.xml}     | 14 +++++-----
 .../scripts/pdt_thread_status.sh}             |  3 ++-
 .../delius/failed_prf_thread/mea.xml          |  6 -----
 9 files changed, 79 insertions(+), 16 deletions(-)
 create mode 100644 ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/collection/ME#24#ALFRESCO_QUEUE_LAG.xml
 create mode 100644 ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/mea.xml
 create mode 100644 ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/metadata/ME#24#ALFRESCO_QUEUE_LAG.xml
 create mode 100644 ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/scripts/alfresco_queue_lag.sh
 rename ansible/roles/oracle-oms-setup/files/metric_extensions/delius/{failed_prf_thread/collection/ME#24#FAILED_PRF_THREAD.xml => failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml} (64%)
 create mode 100644 ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/mea.xml
 rename ansible/roles/oracle-oms-setup/files/metric_extensions/delius/{failed_prf_thread/metadata/ME#24#FAILED_PRF_THREAD.xml => failed_pdt_thread/metadata/ME#24#FAILED_PDT_THREAD.xml} (64%)
 rename ansible/roles/oracle-oms-setup/files/metric_extensions/delius/{failed_prf_thread/scripts/prf_thread_status.sh => failed_pdt_thread/scripts/pdt_thread_status.sh} (72%)
 delete mode 100644 ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/mea.xml

diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/collection/ME#24#ALFRESCO_QUEUE_LAG.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/collection/ME#24#ALFRESCO_QUEUE_LAG.xml
new file mode 100644
index 000000000..c17d19a53
--- /dev/null
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/collection/ME#24#ALFRESCO_QUEUE_LAG.xml
@@ -0,0 +1,9 @@
+<TargetCollectionExt EXT_NAME="ME$ALFRESCO_QUEUE_LAG" EXT_VERSION="1" TARGET_TYPE="host"><CollectionItem NAME="ME$ALFRESCO_QUEUE_LAG" UPLOAD="YES">
+	<Schedule>
+		<IntervalSchedule INTERVAL="15" TIME_UNIT="Min"/>
+	</Schedule>
+	<MetricColl NAME="ME$ALFRESCO_QUEUE_LAG">
+		<Condition COLUMN_NAME="QUEUE_LAG_MINS" CRITICAL="5" OPERATOR="GT" OCCURRENCES="1" MESSAGE="The Alfresco Queue Lag is %value% minutes." CLEAR_MESSAGE="The Alfresco Queue Lag is %value% minutes."/>
+	</MetricColl>
+</CollectionItem>
+</TargetCollectionExt>
\ No newline at end of file
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/mea.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/mea.xml
new file mode 100644
index 000000000..4952d73de
--- /dev/null
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/mea.xml
@@ -0,0 +1,6 @@
+<MetricExtensionArchive name="MEAME$ALFRESCO_QUEUE_LAG">
+<MetricExtension name="ME$ALFRESCO_QUEUE_LAG" type="host" version="1" is_repos="FALSE" display_name="Alfresco Queue Lag" description="Maximum lag time in minutes that Alfresco messages have been queued." version_comment=" " metadata_digest="700b97eaa5bfd3d8d34101ca9acc6cbd" coll_digest="cc16bb83059a4ec23d7db92c5d9d3d36" attach_digest="512e2bf8ef982ed11ad02af109fa31a6">
+<scripts name="alfresco_queue_lag.sh">
+</scripts>
+</MetricExtension>
+</MetricExtensionArchive>
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/metadata/ME#24#ALFRESCO_QUEUE_LAG.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/metadata/ME#24#ALFRESCO_QUEUE_LAG.xml
new file mode 100644
index 000000000..df5a25d36
--- /dev/null
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/metadata/ME#24#ALFRESCO_QUEUE_LAG.xml
@@ -0,0 +1,26 @@
+<TargetMetadataExt EXT_NAME="ME$ALFRESCO_QUEUE_LAG" EXT_VERSION="1" TARGET_TYPE="host"><Metric NAME="ME$ALFRESCO_QUEUE_LAG" TYPE="TABLE">
+<Display>
+<Label NLSID="NLS_METRIC_hostME$ALFRESCO_QUEUE_LAG">Alfresco Queue Lag</Label>
+<Description NLSID="NLS_DESCRIPTION_hostME$ALFRESCO_QUEUE_LAG">Maximum lag time in minutes that Alfresco messages have been queued.</Description>
+</Display>
+<TableDescriptor>
+<ColumnDescriptor NAME="QUEUE_LAG_MINS" TYPE="NUMBER">
+<Display>
+<Label NLSID="NLS_COLUMN_hostME$ALFRESCO_QUEUE_LAGQUEUE_LAG_MINS">Queue Lag Minutes</Label>
+<Unit NLSID="EM_SYS_STANDARD_RATE_MINUTE">MINUTE</Unit>
+<UnitCategory>RATE</UnitCategory>
+</Display>
+<CategoryValue CLASS="Default" CATEGORY_NAME="Capacity">
+</CategoryValue>
+</ColumnDescriptor>
+</TableDescriptor>
+<QueryDescriptor FETCHLET_ID="OSLineToken">
+<Property NAME="command" SCOPE="GLOBAL">/bin/bash</Property>
+<Property NAME="script" SCOPE="GLOBAL" OPTIONAL="TRUE">%scriptsDir%/alfresco_queue_lag.sh</Property>
+<Property NAME="scriptLoc" SCOPE="GLOBAL" OPTIONAL="TRUE">%scriptsDir%</Property>
+<Property NAME="delimiter" SCOPE="GLOBAL" OPTIONAL="TRUE">|</Property>
+<Property NAME="ENVEM_TARGET_NAME" SCOPE="INSTANCE" OPTIONAL="TRUE">NAME</Property>
+<CredentialRef NAME="OSCreds"></CredentialRef>
+</QueryDescriptor>
+</Metric>
+</TargetMetadataExt>
\ No newline at end of file
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/scripts/alfresco_queue_lag.sh b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/scripts/alfresco_queue_lag.sh
new file mode 100644
index 000000000..c96cd861b
--- /dev/null
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/alfresco_queue_lag/scripts/alfresco_queue_lag.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Calculate the maximum number of minutes an alfresco
+# message has been waiting to be processed.
+
+. ~/.bash_profile
+
+# Exit without failure if database is not up
+srvctl status database -d $ORACLE_SID >/dev/null || exit 0
+
+sqlplus -s / as sysdba <<EOSQL
+SET ECHO OFF
+SET FEEDBACK OFF
+SET HEAD OFF
+SET PAGES 0
+SELECT ROUND(COALESCE(MAX(SYSDATE-date_created),0)*24*60,1) unprocessed_mins_ago
+FROM   delius_app_schema.spg_notification
+WHERE  processed_flag=4
+;
+EXIT
+EOSQL
\ No newline at end of file
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/collection/ME#24#FAILED_PRF_THREAD.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml
similarity index 64%
rename from ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/collection/ME#24#FAILED_PRF_THREAD.xml
rename to ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml
index b791640de..136867725 100644
--- a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/collection/ME#24#FAILED_PRF_THREAD.xml
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml
@@ -1,8 +1,8 @@
-<TargetCollectionExt EXT_NAME="ME$FAILED_PRF_THREAD" EXT_VERSION="2" TARGET_TYPE="host"><CollectionItem NAME="ME$FAILED_PRF_THREAD" UPLOAD="YES">
+<TargetCollectionExt EXT_NAME="ME$FAILED_PDT_THREAD" EXT_VERSION="1" TARGET_TYPE="host"><CollectionItem NAME="ME$FAILED_PDT_THREAD" UPLOAD="YES">
 	<Schedule>
 		<IntervalSchedule INTERVAL="1" TIME_UNIT="Hr"/>
 	</Schedule>
-	<MetricColl NAME="ME$FAILED_PRF_THREAD">
+	<MetricColl NAME="ME$FAILED_PDT_THREAD">
 		<Condition COLUMN_NAME="Status" CRITICAL="2" OPERATOR="EQ" OCCURRENCES="1" MESSAGE="Failed PRF thread for %keyValue%" CLEAR_MESSAGE="No failed PRF threads"/>
 	</MetricColl>
 </CollectionItem>
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/mea.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/mea.xml
new file mode 100644
index 000000000..22b9e937c
--- /dev/null
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/mea.xml
@@ -0,0 +1,6 @@
+<MetricExtensionArchive name="MEAME$FAILED_PDT_THREAD">
+<MetricExtension name="ME$FAILED_PDT_THREAD" type="host" version="1" is_repos="FALSE" display_name="Failed PDT Thread" description="Detect failure of Parallel Data Threads" version_comment=" " metadata_digest="83c0ffbab5a51720b3e3e7029e918d9f" coll_digest="366bab14a648562fffc7571eac62ef27" attach_digest="cd0b479c50ca3631d2b62637c31baa41">
+<scripts name="pdt_thread_status.sh">
+</scripts>
+</MetricExtension>
+</MetricExtensionArchive>
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/metadata/ME#24#FAILED_PRF_THREAD.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/metadata/ME#24#FAILED_PDT_THREAD.xml
similarity index 64%
rename from ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/metadata/ME#24#FAILED_PRF_THREAD.xml
rename to ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/metadata/ME#24#FAILED_PDT_THREAD.xml
index e6062f4b6..2decb2fcc 100644
--- a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/metadata/ME#24#FAILED_PRF_THREAD.xml
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/metadata/ME#24#FAILED_PDT_THREAD.xml
@@ -1,22 +1,22 @@
-<TargetMetadataExt EXT_NAME="ME$FAILED_PRF_THREAD" EXT_VERSION="2" TARGET_TYPE="host"><Metric NAME="ME$FAILED_PRF_THREAD" TYPE="TABLE">
+<TargetMetadataExt EXT_NAME="ME$FAILED_PDT_THREAD" EXT_VERSION="1" TARGET_TYPE="host"><Metric NAME="ME$FAILED_PDT_THREAD" TYPE="TABLE">
 <Display>
-<Label NLSID="NLS_METRIC_hostME$FAILED_PRF_THREAD">Failed PRF Thread</Label>
-<Description NLSID="NLS_DESCRIPTION_hostME$FAILED_PRF_THREAD">Detect failure of custom performance monitoring threads</Description>
+<Label NLSID="NLS_METRIC_hostME$FAILED_PDT_THREAD">Failed PDT Thread</Label>
+<Description NLSID="NLS_DESCRIPTION_hostME$FAILED_PDT_THREAD">Detect failure of Parallel Data Threads</Description>
 </Display>
 <TableDescriptor>
 <ColumnDescriptor NAME="COMPONENT_ID" TYPE="NUMBER" IS_KEY="TRUE">
 <Display>
-<Label NLSID="NLS_COLUMN_hostME$FAILED_PRF_THREADCOMPONENT_ID">Component ID</Label>
+<Label NLSID="NLS_COLUMN_hostME$FAILED_PDT_THREADCOMPONENT_ID">Component ID</Label>
 </Display>
 </ColumnDescriptor>
 <ColumnDescriptor NAME="THREAD_ID" TYPE="NUMBER" IS_KEY="TRUE">
 <Display>
-<Label NLSID="NLS_COLUMN_hostME$FAILED_PRF_THREADTHREAD_ID">Thread ID</Label>
+<Label NLSID="NLS_COLUMN_hostME$FAILED_PDT_THREADTHREAD_ID">Thread ID</Label>
 </Display>
 </ColumnDescriptor>
 <ColumnDescriptor NAME="Status" TYPE="NUMBER">
 <Display>
-<Label NLSID="NLS_COLUMN_hostME$FAILED_PRF_THREADStatus">Thread Status</Label>
+<Label NLSID="NLS_COLUMN_hostME$FAILED_PDT_THREADStatus">Thread Status</Label>
 </Display>
 <CategoryValue CLASS="Default" CATEGORY_NAME="Fault">
 </CategoryValue>
@@ -24,7 +24,7 @@
 </TableDescriptor>
 <QueryDescriptor FETCHLET_ID="OSLineToken">
 <Property NAME="command" SCOPE="GLOBAL">/bin/bash</Property>
-<Property NAME="script" SCOPE="GLOBAL" OPTIONAL="TRUE">%scriptsDir%/prf_thread_status.sh</Property>
+<Property NAME="script" SCOPE="GLOBAL" OPTIONAL="TRUE">%scriptsDir%/pdt_thread_status.sh</Property>
 <Property NAME="scriptLoc" SCOPE="GLOBAL" OPTIONAL="TRUE">%scriptsDir%</Property>
 <Property NAME="delimiter" SCOPE="GLOBAL" OPTIONAL="TRUE">|</Property>
 <Property NAME="ENVEM_TARGET_NAME" SCOPE="INSTANCE" OPTIONAL="TRUE">NAME</Property>
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/scripts/prf_thread_status.sh b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/scripts/pdt_thread_status.sh
similarity index 72%
rename from ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/scripts/prf_thread_status.sh
rename to ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/scripts/pdt_thread_status.sh
index fb2d715a6..d466a6661 100644
--- a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/scripts/prf_thread_status.sh
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/scripts/pdt_thread_status.sh
@@ -15,7 +15,8 @@ SET HEAD OFF
 SET PAGES 0
 SELECT component_id||'|'||thread_id||'|'||status
 FROM delius_app_schema.pdt_thread 
-WHERE component_id BETWEEN 500 AND 505
+WHERE component_id BETWEEN 500 AND 505   -- Performance Monitoring Threads
+OR         component_id BETWEEN 100 AND 107   -- Alfresco Message Queue Threads
 ORDER BY component_id, thread_id;
 EXIT
 EOSQL
\ No newline at end of file
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/mea.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/mea.xml
deleted file mode 100644
index c3e9bb669..000000000
--- a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_prf_thread/mea.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-<MetricExtensionArchive name="MEAME$FAILED_PRF_THREAD">
-<MetricExtension name="ME$FAILED_PRF_THREAD" type="host" version="2" is_repos="FALSE" display_name="Failed PRF Thread" description="Detect failure of custom performance monitoring threads" version_comment=" " metadata_digest="4702aa08f8e9cc972097929a008ac642" coll_digest="33cfed30f14a4d72882ce0c9c821c43d" attach_digest="05d87307ff694aa5859708259e7e730b">
-<scripts name="prf_thread_status.sh">
-</scripts>
-</MetricExtension>
-</MetricExtensionArchive>

From 281093d22fcf8ad48ebfe95e28ca4cbe7a5f7aaf Mon Sep 17 00:00:00 2001
From: Bill Buchan <bill.buchan@digital.justice.gov.uk>
Date: Tue, 14 May 2024 14:30:13 +0100
Subject: [PATCH 2/2] Fix naming in error message

---
 .../failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml  | 2 +-
 .../delius/failed_pdt_thread/scripts/pdt_thread_status.sh     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml
index 136867725..9a6cd4a7b 100644
--- a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/collection/ME#24#FAILED_PDT_THREAD.xml
@@ -3,7 +3,7 @@
 		<IntervalSchedule INTERVAL="1" TIME_UNIT="Hr"/>
 	</Schedule>
 	<MetricColl NAME="ME$FAILED_PDT_THREAD">
-		<Condition COLUMN_NAME="Status" CRITICAL="2" OPERATOR="EQ" OCCURRENCES="1" MESSAGE="Failed PRF thread for %keyValue%" CLEAR_MESSAGE="No failed PRF threads"/>
+		<Condition COLUMN_NAME="Status" CRITICAL="2" OPERATOR="EQ" OCCURRENCES="1" MESSAGE="Failed PDT thread for %keyValue%" CLEAR_MESSAGE="No failed PRF threads"/>
 	</MetricColl>
 </CollectionItem>
 </TargetCollectionExt>
\ No newline at end of file
diff --git a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/scripts/pdt_thread_status.sh b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/scripts/pdt_thread_status.sh
index d466a6661..634945e49 100644
--- a/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/scripts/pdt_thread_status.sh
+++ b/ansible/roles/oracle-oms-setup/files/metric_extensions/delius/failed_pdt_thread/scripts/pdt_thread_status.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-#  Get status code for PRF threads (2=failed)
+#  Get status code for PDT threads (2=failed)
 #
 
 . ~/.bash_profile
@@ -15,7 +15,7 @@ SET HEAD OFF
 SET PAGES 0
 SELECT component_id||'|'||thread_id||'|'||status
 FROM delius_app_schema.pdt_thread 
-WHERE component_id BETWEEN 500 AND 505   -- Performance Monitoring Threads
+WHERE component_id BETWEEN 500 AND 505   -- Performance Monitoring Threads (PRF)
 OR         component_id BETWEEN 100 AND 107   -- Alfresco Message Queue Threads
 ORDER BY component_id, thread_id;
 EXIT