diff --git a/ansible/check_pods_status.yml b/ansible/check_pods_status.yml new file mode 100644 index 0000000000..0247937f64 --- /dev/null +++ b/ansible/check_pods_status.yml @@ -0,0 +1,22 @@ +--- +- name: Check Pod Status in All Namespaces + hosts: localhost + gather_facts: no + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + tasks: + - name: Check Non-Running Pods in Namespaces + shell: kubectl get pods -A | grep -v Running + register: non_running_pods + changed_when: false + failed_when: false + + - name: Parse Non-Running Pods + set_fact: + non_running_pod_lines: "{{ non_running_pods.stdout_lines }}" + when: non_running_pods.stdout_lines | length > 0 + + - name: Print Non-Running Pods + debug: + msg: "{{ non_running_pod_lines | join('\n') }}" + when: non_running_pod_lines is defined diff --git a/ansible/check_services.yml b/ansible/check_services.yml new file mode 100644 index 0000000000..b4c2a65ab2 --- /dev/null +++ b/ansible/check_services.yml @@ -0,0 +1,106 @@ +- hosts: cassandra + tasks: + - name: Check Cassandra service status + service: + name: cassandra + state: started + register: cassandra_status + +- hosts: postgres + tasks: + - name: Check postgres service status + service: + name: postgresql@12-main.service + state: started + register: postgres_status + # - name: Start postgres service if it's not running + # service: + # name: postgresql@12-main.service + # state: started + # when: postgres_status.changed + # become: true + +- hosts: learning-neo4j-node1 + tasks: + - name: Check Neo4j Service Status + command: "/home/learning/neo4j-learning/neo4j-enterprise-3.3.10-SNAPSHOT/bin/neo4j status" + register: service_status + changed_when: false + become: true + +- hosts: composite-search-cluster + tasks: + - name: Check Elasticsearch Port + command: "nc -z -v -w 2 {{ inventory_hostname }} 9200" + ignore_errors: yes + register: nc_result + +- hosts: lp-redis-ps + tasks: + - name: Check redis service status + service: + name: redis + state: started + register: redis_status + #- name: Start redis service if it's not running + # service: + # name: redis + # state: started + # when: redis_status.changed + # become: true + +- hosts: kafka + tasks: + - name: Check kafka service status + service: + name: kafka + state: started + register: kafka_status + #- name: Start kafka service if it's not running + # service: + # name: kafka + # state: started + # when: kafka_status.changed + # become: true + +- hosts: keycloak + tasks: + - name: Check keycloak service status + service: + name: keycloak + state: started + register: keycloak_status + #- name: Start keycloak service if it's not running + # service: + # name: keycloak + # state: started + # when: keycloak_status.changed + # become: true + +- hosts: graylog + tasks: + - name: Check graylog service status + service: + name: graylog-server + state: started + register: graylog_status + #- name: Start graylog service if it's not running + # service: + # name: graylog-server + # state: started + # when: graylog_status.changed + # become: true + +- hosts: lp-learning-ps + tasks: + - name: Check Tomcat Port (learning VM) + command: "nc -z -v -w 2 {{ inventory_hostname }} 8080" + ignore_errors: yes + register: nc_tomcat_result + +- hosts: internal-proxy + tasks: + - name: Check Kong Port (Internal proxy) + command: "nc -z -v -w 2 {{ inventory_hostname }} 8000" + ignore_errors: yes + register: nc_kong_result diff --git a/pipelines/ops/check-pod-status/Jenkinsfile b/pipelines/ops/check-pod-status/Jenkinsfile new file mode 100644 index 0000000000..cb1ff03989 --- /dev/null +++ b/pipelines/ops/check-pod-status/Jenkinsfile @@ -0,0 +1,50 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/check_pods_status.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/ops/check-service-status/Jenkinsfile b/pipelines/ops/check-service-status/Jenkinsfile new file mode 100644 index 0000000000..07c1395be2 --- /dev/null +++ b/pipelines/ops/check-service-status/Jenkinsfile @@ -0,0 +1,50 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + stage('deploy'){ + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "${currentWs}/ansible/check_services.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +}