From dbe0c696b06b03d86e2df201722675c2e763afa3 Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Fri, 8 Sep 2023 19:45:12 +0530 Subject: [PATCH 1/6] Add playbook to check status of the services --- ansible/check_services.yml | 92 +++++++++++++++++++ .../ops/check-service-status/Jenkinsfile | 50 ++++++++++ 2 files changed, 142 insertions(+) create mode 100644 ansible/check_services.yml create mode 100644 pipelines/ops/check-service-status/Jenkinsfile diff --git a/ansible/check_services.yml b/ansible/check_services.yml new file mode 100644 index 0000000000..d22406583b --- /dev/null +++ b/ansible/check_services.yml @@ -0,0 +1,92 @@ +- hosts: cassandra + tasks: + - name: Check Cassandra service status + service: + name: cassandra + state: started + register: cassandra_status + +- hosts: postgres + tasks: + - name: Check postgres service status + service: + name: postgresql@12-main.service + state: started + register: postgres_status + - name: Start postgres service if it's not running + service: + name: postgresql@12-main.service + state: started + when: postgres_status.changed + become: true + +- hosts: neo4j-ps + tasks: + - name: Check Neo4j Service Status + command: "/home/learning/neo4j-learning/neo4j-enterprise-3.3.10-SNAPSHOT/bin/neo4j status" + register: service_status + changed_when: false + become: true + +- hosts: composite-search-cluster + tasks: + - name: Check Elasticsearch Port + command: "nc -z -v -w 2 {{ inventory_hostname }} 9200" + ignore_errors: yes + register: nc_result + +- hosts: redis-cluster + tasks: + - name: Check redis service status + service: + name: redis + state: started + register: redis_status + - name: Start redis service if it's not running + service: + name: redis + state: started + when: redis_status.changed + become: true + +- hosts: kafka + tasks: + - name: Check kafka service status + service: + name: kafka + state: started + register: kafka_status + - name: Start kafka service if it's not running + service: + name: kafka + state: started + when: kafka_status.changed + become: true + +- hosts: keycloak + tasks: + - name: Check keycloak service status + service: + name: keycloak + state: started + register: keycloak_status + - name: Start keycloak service if it's not running + service: + name: keycloak + state: started + when: keycloak_status.changed + become: true + +- hosts: graylog + tasks: + - name: Check graylog service status + service: + name: graylog-server + state: started + register: graylog_status + - name: Start graylog service if it's not running + service: + name: graylog-server + state: started + when: graylog_status.changed + become: true diff --git a/pipelines/ops/check-service-status/Jenkinsfile b/pipelines/ops/check-service-status/Jenkinsfile new file mode 100644 index 0000000000..07c1395be2 --- /dev/null +++ b/pipelines/ops/check-service-status/Jenkinsfile @@ -0,0 +1,50 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/check_services.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} From f8c1811d889d569ec6ba182ed2c135d98c3e449a Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Fri, 8 Sep 2023 19:52:46 +0530 Subject: [PATCH 2/6] Add tomcat service check --- ansible/check_services.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/check_services.yml b/ansible/check_services.yml index d22406583b..528de3cb60 100644 --- a/ansible/check_services.yml +++ b/ansible/check_services.yml @@ -90,3 +90,10 @@ state: started when: graylog_status.changed become: true + +- hosts: learning1 + tasks: + - name: Check Tomcat Port (learning VM) + command: "nc -z -v -w 2 {{ inventory_hostname }} 8080" + ignore_errors: yes + register: nc_tomcat_result From e12ac550b51024c43490888d415579d0687975e9 Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Mon, 11 Sep 2023 15:48:04 +0530 Subject: [PATCH 3/6] Add playbook to check pod status --- ansible/check_pods_status.yml | 23 ++++++++++ pipelines/ops/check-pod-status/Jenkinsfile | 50 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 ansible/check_pods_status.yml create mode 100644 pipelines/ops/check-pod-status/Jenkinsfile diff --git a/ansible/check_pods_status.yml b/ansible/check_pods_status.yml new file mode 100644 index 0000000000..8c9573c18c --- /dev/null +++ b/ansible/check_pods_status.yml @@ -0,0 +1,23 @@ +-- +- name: Check Pod Status in All Namespaces + hosts: localhost + gather_facts: no + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + + tasks: + - name: Check Non-Running Pods in Namespaces + shell: kubectl get pods -A | grep -v Running + register: non_running_pods + changed_when: false + failed_when: false + + - name: Parse Non-Running Pods + set_fact: + non_running_pod_lines: "{{ non_running_pods.stdout_lines }}" + when: non_running_pods.stdout_lines | length > 0 + + - name: Print Non-Running Pods + debug: + msg: "{{ non_running_pod_lines | join('\n') }}" + when: non_running_pod_lines is defined diff --git a/pipelines/ops/check-pod-status/Jenkinsfile b/pipelines/ops/check-pod-status/Jenkinsfile new file mode 100644 index 0000000000..cb1ff03989 --- /dev/null +++ b/pipelines/ops/check-pod-status/Jenkinsfile @@ -0,0 +1,50 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/check_pods_status.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} From f9e5b281b49d63aff6b63db53ebe62c201f61899 Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Tue, 12 Sep 2023 10:33:38 +0530 Subject: [PATCH 4/6] Fix Syntax --- ansible/check_pods_status.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/check_pods_status.yml b/ansible/check_pods_status.yml index 8c9573c18c..0247937f64 100644 --- a/ansible/check_pods_status.yml +++ b/ansible/check_pods_status.yml @@ -1,10 +1,9 @@ --- +--- - name: Check Pod Status in All Namespaces hosts: localhost gather_facts: no environment: KUBECONFIG: "{{ kubeconfig_path }}" - tasks: - name: Check Non-Running Pods in Namespaces shell: kubectl get pods -A | grep -v Running From 18786332c5bb818d9668f6bd23bab4148301df8e Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Tue, 12 Sep 2023 12:46:28 +0530 Subject: [PATCH 5/6] Add internal-proxy status check --- ansible/check_services.yml | 69 +++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/ansible/check_services.yml b/ansible/check_services.yml index 528de3cb60..a0c9a1edda 100644 --- a/ansible/check_services.yml +++ b/ansible/check_services.yml @@ -13,12 +13,12 @@ name: postgresql@12-main.service state: started register: postgres_status - - name: Start postgres service if it's not running - service: - name: postgresql@12-main.service - state: started - when: postgres_status.changed - become: true + # - name: Start postgres service if it's not running + # service: + # name: postgresql@12-main.service + # state: started + # when: postgres_status.changed + # become: true - hosts: neo4j-ps tasks: @@ -35,19 +35,19 @@ ignore_errors: yes register: nc_result -- hosts: redis-cluster +- hosts: lp-redis tasks: - name: Check redis service status service: name: redis state: started register: redis_status - - name: Start redis service if it's not running - service: - name: redis - state: started - when: redis_status.changed - become: true + #- name: Start redis service if it's not running + # service: + # name: redis + # state: started + # when: redis_status.changed + # become: true - hosts: kafka tasks: @@ -56,12 +56,12 @@ name: kafka state: started register: kafka_status - - name: Start kafka service if it's not running - service: - name: kafka - state: started - when: kafka_status.changed - become: true + #- name: Start kafka service if it's not running + # service: + # name: kafka + # state: started + # when: kafka_status.changed + # become: true - hosts: keycloak tasks: @@ -70,12 +70,12 @@ name: keycloak state: started register: keycloak_status - - name: Start keycloak service if it's not running - service: - name: keycloak - state: started - when: keycloak_status.changed - become: true + #- name: Start keycloak service if it's not running + # service: + # name: keycloak + # state: started + # when: keycloak_status.changed + # become: true - hosts: graylog tasks: @@ -84,12 +84,12 @@ name: graylog-server state: started register: graylog_status - - name: Start graylog service if it's not running - service: - name: graylog-server - state: started - when: graylog_status.changed - become: true + #- name: Start graylog service if it's not running + # service: + # name: graylog-server + # state: started + # when: graylog_status.changed + # become: true - hosts: learning1 tasks: @@ -97,3 +97,10 @@ command: "nc -z -v -w 2 {{ inventory_hostname }} 8080" ignore_errors: yes register: nc_tomcat_result + +- hosts: internal-proxy + tasks: + - name: Check Kong Port (Internal proxy) + command: "nc -z -v -w 2 {{ inventory_hostname }} 8080" + ignore_errors: yes + register: nc_kong_result From cd3d436ec315d6c8198f5750af8e447dd68feb6a Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Tue, 12 Sep 2023 16:33:35 +0530 Subject: [PATCH 6/6] Update hosts --- ansible/check_services.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/check_services.yml b/ansible/check_services.yml index a0c9a1edda..b4c2a65ab2 100644 --- a/ansible/check_services.yml +++ b/ansible/check_services.yml @@ -20,7 +20,7 @@ # when: postgres_status.changed # become: true -- hosts: neo4j-ps +- hosts: learning-neo4j-node1 tasks: - name: Check Neo4j Service Status command: "/home/learning/neo4j-learning/neo4j-enterprise-3.3.10-SNAPSHOT/bin/neo4j status" @@ -35,7 +35,7 @@ ignore_errors: yes register: nc_result -- hosts: lp-redis +- hosts: lp-redis-ps tasks: - name: Check redis service status service: @@ -91,7 +91,7 @@ # when: graylog_status.changed # become: true -- hosts: learning1 +- hosts: lp-learning-ps tasks: - name: Check Tomcat Port (learning VM) command: "nc -z -v -w 2 {{ inventory_hostname }} 8080" @@ -101,6 +101,6 @@ - hosts: internal-proxy tasks: - name: Check Kong Port (Internal proxy) - command: "nc -z -v -w 2 {{ inventory_hostname }} 8080" + command: "nc -z -v -w 2 {{ inventory_hostname }} 8000" ignore_errors: yes register: nc_kong_result