[ED-2837] Add scripts to check services and pod running status

ansible/check_pods_status.yml

@@ -0,0 +1,22 @@
+---
+- name: Check Pod Status in All Namespaces
+  hosts: localhost
+  gather_facts: no
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+  tasks:
+    - name: Check Non-Running Pods in Namespaces
+      shell: kubectl get pods -A | grep -v Running
+      register: non_running_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Parse Non-Running Pods
+      set_fact:
+        non_running_pod_lines: "{{ non_running_pods.stdout_lines }}"
+      when: non_running_pods.stdout_lines | length > 0
+
+    - name: Print Non-Running Pods
+      debug:
+        msg: "{{ non_running_pod_lines | join('\n') }}"
+      when: non_running_pod_lines is defined

ansible/check_services.yml

@@ -0,0 +1,106 @@
+- hosts: cassandra  
+  tasks:
+    - name: Check Cassandra service status
+      service:
+        name: cassandra
+        state: started
+      register: cassandra_status
+
+- hosts: postgres
+  tasks:
+    - name: Check postgres service status
+      service:
+        name: [email protected]
+        state: started
+      register: postgres_status
+   # - name: Start postgres service if it's not running
+   #   service:
+   #     name: [email protected]
+   #     state: started
+   #   when: postgres_status.changed
+   #   become: true
+
+- hosts: learning-neo4j-node1
+  tasks:
+    - name: Check Neo4j Service Status
+      command: "/home/learning/neo4j-learning/neo4j-enterprise-3.3.10-SNAPSHOT/bin/neo4j status"
+      register: service_status
+      changed_when: false
+      become: true
+
+- hosts: composite-search-cluster
+  tasks:
+    - name: Check Elasticsearch Port
+      command: "nc -z -v -w 2 {{ inventory_hostname }} 9200"
+      ignore_errors: yes
+      register: nc_result
+
+- hosts: lp-redis-ps
+  tasks:
+    - name: Check redis service status
+      service:
+        name: redis
+        state: started
+      register: redis_status
+    #- name: Start redis service if it's not running
+    #  service:
+    #    name: redis
+    #    state: started
+    #  when: redis_status.changed
+    #  become: true
+
+- hosts: kafka
+  tasks:
+    - name: Check kafka service status
+      service:
+        name: kafka
+        state: started
+      register: kafka_status
+    #- name: Start kafka service if it's not running
+    #  service:
+    #    name: kafka
+    #    state: started
+    #  when: kafka_status.changed
+    #  become: true
+
+- hosts: keycloak
+  tasks:
+    - name: Check keycloak service status
+      service:
+        name: keycloak
+        state: started
+      register: keycloak_status
+    #- name: Start keycloak service if it's not running
+    #  service:
+    #    name: keycloak
+    #    state: started
+    #  when: keycloak_status.changed
+    #  become: true
+
+- hosts: graylog
+  tasks:
+    - name: Check graylog service status
+      service:
+        name: graylog-server
+        state: started
+      register: graylog_status
+    #- name: Start graylog service if it's not running
+    #  service:
+    #    name: graylog-server
+    #    state: started
+    #  when: graylog_status.changed
+    #  become: true
+
+- hosts: lp-learning-ps
+  tasks:
+    - name: Check Tomcat Port (learning VM)
+      command: "nc -z -v -w 2 {{ inventory_hostname }} 8080"
+      ignore_errors: yes
+      register: nc_tomcat_result
+
+- hosts: internal-proxy
+  tasks:
+    - name: Check Kong Port (Internal proxy)
+      command: "nc -z -v -w 2 {{ inventory_hostname }} 8000"
+      ignore_errors: yes
+      register: nc_kong_result

pipelines/ops/check-pod-status/Jenkinsfile

@@ -0,0 +1,50 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/check_pods_status.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+                currentBuild.result = 'SUCCESS'
+                currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+    }
+    catch (err) {
+        currentBuild.result = 'FAILURE'
+        throw err
+    }    
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}

pipelines/ops/check-service-status/Jenkinsfile

@@ -0,0 +1,50 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/check_services.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+                currentBuild.result = 'SUCCESS'
+                currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+    }
+    catch (err) {
+        currentBuild.result = 'FAILURE'
+        throw err
+    }    
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}