Mongo backup (#2412)

Mongodb backup role
project-sunbird · May 3, 2021 · 806bf76 · 806bf76
1 parent a64156e
commit 806bf76
Show file tree

Hide file tree

Showing 6 changed files with 214 additions and 0 deletions.
diff --git a/ansible/mongodb-backup.yml b/ansible/mongodb-backup.yml
@@ -0,0 +1,9 @@
+- hosts: "{{ host }}"
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml']
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}"
+  roles:
+   - mongodb-backup
diff --git a/ansible/roles/mongodb-backup/defaults/main.yml b/ansible/roles/mongodb-backup/defaults/main.yml
@@ -0,0 +1,2 @@
+mongo_backup_dir: '/tmp/mongo-backup'
+mongo_backup_azure_container_name: "{{ mongo_backup_azure_container_name }}"
diff --git a/ansible/roles/mongodb-backup/meta/main.yml b/ansible/roles/mongodb-backup/meta/main.yml
@@ -0,0 +1,2 @@
+dependencies:
+  - azure-cli
diff --git a/ansible/roles/mongodb-backup/tasks/main.yml b/ansible/roles/mongodb-backup/tasks/main.yml
@@ -0,0 +1,25 @@
+- name: Create the directory
+  file: path={{ mongo_backup_dir }} state=directory recurse=yes
+
+- set_fact:
+    mongo_backup_file_name: "mongo-backup-{{ ansible_hostname }}-{{ lookup('pipe', 'date +%Y-%m-%d') }}"
+
+- set_fact:
+    mongo_backup_file_path: "{{ mongo_backup_dir }}/{{ mongo_backup_file_name }}"
+
+- name: take dump
+  shell: "mongodump --out={{ mongo_backup_file_path }}"
+
+- name: Compress the backup file
+  shell: "tar -czf {{ mongo_backup_file_path }}.tar.gz {{ mongo_backup_file_path }}"
+
+- name: upload to azure
+  include_role:
+    name: artifacts-upload-azure
+  vars:
+    artifact: "{{ mongo_backup_file_name }}.tar.gz"
+    artifact_path: "{{ mongo_backup_file_path }}.tar.gz"
+    artifacts_container: "{{ mongo_backup_azure_container_name }}"
+
+- name: clean up backup dir after upload
+  file: path={{ mongo_backup_dir }} state=absent
diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/MongoBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/MongoBackup/config.xml
@@ -0,0 +1,130 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="[email protected]">
+  <actions/>
+  <description/>
+  <keepDependencies>false</keepDependencies>
+  <properties>
+    <jenkins.model.BuildDiscarderProperty>
+      <strategy class="hudson.tasks.LogRotator">
+        <daysToKeep>-1</daysToKeep>
+        <numToKeep>-1</numToKeep>
+        <artifactDaysToKeep>-1</artifactDaysToKeep>
+        <artifactNumToKeep>5</artifactNumToKeep>
+      </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+    <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+    <com.sonyericsson.rebuild.RebuildSettings plugin="[email protected]">
+      <autoRebuild>false</autoRebuild>
+      <rebuildDisabled>false</rebuildDisabled>
+    </com.sonyericsson.rebuild.RebuildSettings>
+    <hudson.model.ParametersDefinitionProperty>
+      <parameterDefinitions>
+        <org.biouno.unochoice.DynamicReferenceParameter plugin="[email protected]">
+          <name>private_branch</name>
+          <description/>
+          <randomName>choice-parameter-189747244589805</randomName>
+          <visibleItemCount>1</visibleItemCount>
+          <script class="org.biouno.unochoice.model.GroovyScript">
+            <secureScript plugin="[email protected]">
+              <script>if (override_private_branch.equals("true")) {
+return """&lt;input name="value" value="${private_repo_branch}" class="setting-input"  type="text"&gt;&lt;br&gt; &lt;font color=dimgray size=2&gt;&lt;b&gt;Change this value to checkout a different branch from private repository.&lt;/b&gt;&lt;/font&gt;"""
+}
+else
+return """&lt;b&gt;This parameter is not used&lt;/b&gt;"""</script>
+              <sandbox>false</sandbox>
+            </secureScript>
+            <secureFallbackScript plugin="[email protected]">
+              <script>return """&lt;b&gt;This parameter is not used&lt;/b&gt;"""</script>
+              <sandbox>false</sandbox>
+            </secureFallbackScript>
+          </script>
+          <projectName>Mongodb</projectName>
+          <projectFullName>OpsAdministration/dev/Core/Mongodb</projectFullName>
+          <parameters class="linked-hash-map"/>
+          <referencedParameters/>
+          <choiceType>ET_FORMATTED_HTML</choiceType>
+          <omitValueField>true</omitValueField>
+        </org.biouno.unochoice.DynamicReferenceParameter>
+        <org.biouno.unochoice.DynamicReferenceParameter plugin="[email protected]">
+          <name>branch_or_tag</name>
+          <description/>
+          <randomName>choice-parameter-189747247305087</randomName>
+          <visibleItemCount>1</visibleItemCount>
+          <script class="org.biouno.unochoice.model.GroovyScript">
+            <secureScript plugin="[email protected]">
+              <script>if (override_public_branch.equals("true")) {
+return """&lt;input name="value" value="" class="setting-input"  type="text"&gt;&lt;br&gt; &lt;font color=dimgray size=2&gt;&lt;b&gt;Provide the tag or branch name to checkout the Jenkinsfile and codebase.&lt;br&gt;Note: The tag or branch name for this job should be taken from &lt;a href="https://github.com/project-sunbird/sunbird-devops"&gt;project-sunbird/sunbird-devops&lt;/a&gt;&lt;/b&gt;&lt;/font&gt;&lt;/b&gt;&lt;/font&gt;""";
+
+}
+else
+return """&lt;b&gt;This parameter is not used&lt;/b&gt;"""</script>
+              <sandbox>false</sandbox>
+            </secureScript>
+            <secureFallbackScript plugin="[email protected]">
+              <script>return """&lt;b&gt;This parameter is not used&lt;/b&gt;"""</script>
+              <sandbox>false</sandbox>
+            </secureFallbackScript>
+          </script>
+          <projectName>Mongodb</projectName>
+          <projectFullName>OpsAdministration/dev/Core/Mongodb</projectFullName>
+          <parameters class="linked-hash-map"/>
+          <referencedParameters/>
+          <choiceType>ET_FORMATTED_HTML</choiceType>
+          <omitValueField>true</omitValueField>
+        </org.biouno.unochoice.DynamicReferenceParameter>
+        <hudson.model.StringParameterDefinition>
+          <name>host</name>
+          <description>Host group name of server for which we need backup</description>
+          <defaultValue>mongodb-backup</defaultValue>
+          <trim>false</trim>
+        </hudson.model.StringParameterDefinition>
+      </parameterDefinitions>
+    </hudson.model.ParametersDefinitionProperty>
+    <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="[email protected]">
+      <maxConcurrentPerNode>0</maxConcurrentPerNode>
+      <maxConcurrentTotal>0</maxConcurrentTotal>
+      <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+      <throttleEnabled>false</throttleEnabled>
+      <throttleOption>project</throttleOption>
+      <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+      <paramsToUseForLimit/>
+    </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+    <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+      <triggers>
+        <hudson.triggers.TimerTrigger>
+          <spec>00 1 * * *</spec>
+        </hudson.triggers.TimerTrigger>
+      </triggers>
+    </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+  </properties>
+  <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="[email protected]">
+    <scm class="hudson.plugins.git.GitSCM" plugin="[email protected]">
+      <configVersion>2</configVersion>
+      <userRemoteConfigs>
+        <hudson.plugins.git.UserRemoteConfig>
+          <url>https://github.com/project-sunbird/sunbird-devops.git</url>
+        </hudson.plugins.git.UserRemoteConfig>
+      </userRemoteConfigs>
+      <branches>
+        <hudson.plugins.git.BranchSpec>
+          <name>${branch_or_tag}</name>
+        </hudson.plugins.git.BranchSpec>
+      </branches>
+      <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+      <submoduleCfg class="empty-list"/>
+      <extensions>
+        <hudson.plugins.git.extensions.impl.CloneOption>
+          <shallow>true</shallow>
+          <noTags>false</noTags>
+          <reference/>
+          <depth>0</depth>
+          <honorRefspec>false</honorRefspec>
+        </hudson.plugins.git.extensions.impl.CloneOption>
+      </extensions>
+    </scm>
+    <scriptPath>pipelines/backup/mongodb-backup/Jenkinsfile</scriptPath>
+    <lightweight>false</lightweight>
+  </definition>
+  <triggers/>
+  <disabled>false</disabled>
+</flow-definition>
diff --git a/pipelines/backup/mongodb-backup/Jenkinsfile b/pipelines/backup/mongodb-backup/Jenkinsfile
@@ -0,0 +1,46 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout Private repo') {
+            cleanWs()
+            checkout scm
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/mongodb-backup.yml"
+                ansibleExtraArgs = "--extra-vars \"host=${params.host}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+                currentBuild.result = "SUCCESS"
+                currentBuild.description = "Private: ${params.private_branch}"
+             }
+        }
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		mongo_backup_dir: '/tmp/mongo-backup'
		mongo_backup_azure_container_name: "{{ mongo_backup_azure_container_name }}"