Add ETL pipeline testing QS (e2e-python)

.github/dependabot.yml

@@ -8,6 +8,3 @@ updates:
     schedule:
       # Check for updates to GitHub Actions every week
       interval: "weekly"
-    labels:
-      - "dependencies"
-      - "skip changelog"

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ### Added
+- Added ETL pipeline testing QS (e2e-python) ([#985](https://github.com/opendevstack/ods-quickstarters/pull/985))
 - Update gateway-Nginx quickstarter ([#983](https://github.com/opendevstack/ods-quickstarters/pull/983))
 - Added secret scanning in docker plain ([#963](https://github.com/opendevstack/ods-quickstarters/pull/963))
 - Added Nodejs20 agent ([#962](https://github.com/opendevstack/ods-quickstarters/issues/962))

be-gateway-nginx/files/docker/Dockerfile

@@ -1,5 +1,5 @@
 # https://github.com/openresty/docker-openresty
-FROM openresty/openresty:1.25.3.1-1-rocky
+FROM openresty/openresty:1.21.4.1-rocky
 
 ENV LANG=C.UTF-8
 

be-gateway-nginx/files/metadata.yml

@@ -1,6 +1,6 @@
 ---
 name: nginx
-description: "Enhanced nginx with Lua embeded. nginx [engine x] is an HTTP and reverse proxy server, a mail proxy server, and a generic TCP/UDP proxy server. Technologies: OpenResty/nginx 1.25.3.1-1"
+description: "Enhanced nginx with Lua embeded. nginx [engine x] is an HTTP and reverse proxy server, a mail proxy server, and a generic TCP/UDP proxy server. Technologies: OpenResty/nginx 1.19.3.2"
 supplier: https://openresty.org
 version: 4.x
 type: ods-service

docs/modules/quickstarters/pages/be-gateway-nginx.adoc

@@ -109,9 +109,9 @@ Note that the xref:jenkins-shared-library:labelling.adoc[OpenShift resources wil
 
 ```yaml
 name: nginx
-description: "Enhanced nginx with Lua embeded. nginx [engine x] is an HTTP and reverse proxy server, a mail proxy server, and a generic TCP/UDP proxy server. Technologies: OpenResty/nginx 1.25.3.1-1"
+description: "Enhanced nginx with Lua embeded. nginx [engine x] is an HTTP and reverse proxy server, a mail proxy server, and a generic TCP/UDP proxy server. Technologies: OpenResty/nginx 1.19.3.2"
 supplier: https://openresty.org
-version: 1.25.3.1-1
+version: 1.21.4.1
 type: ods-service
 role: integration
 ```

e2e-python/Jenkinsfile

@@ -0,0 +1,48 @@
+def odsNamespace = ''
+def odsGitRef = ''
+def odsImageTag = ''
+def sharedLibraryRef = ''
+def agentImageTag = ''
+
+node {
+  odsNamespace = env.ODS_NAMESPACE ?: 'ods'
+  odsGitRef = env.ODS_GIT_REF ?: 'master'
+  odsImageTag = env.ODS_IMAGE_TAG ?: 'latest'
+  sharedLibraryRef = env.SHARED_LIBRARY_REF ?: odsImageTag
+  agentImageTag = env.AGENT_IMAGE_TAG ?: odsImageTag
+}
+
+library("ods-jenkins-shared-library@${sharedLibraryRef}")
+
+odsQuickstarterPipeline(
+  imageStreamTag: "${odsNamespace}/jenkins-agent-base:${agentImageTag}",
+) { context ->
+
+  odsQuickstarterStageCopyFiles(context)
+
+  odsQuickstarterStageRenderJenkinsfile(context)
+
+  odsQuickstarterStageRenderJenkinsfile(
+    context,
+    [source: 'dev.yml.template',
+     target: 'environments/dev.yml']
+  )
+
+  odsQuickstarterStageRenderJenkinsfile(
+    context,
+   [source: 'test.yml.template',
+     target: 'environments/test.yml']
+  )
+
+  odsQuickstarterStageRenderJenkinsfile(
+    context,
+    [source: 'prod.yml.template',
+     target: 'environments/prod.yml']
+  )
+
+  odsQuickstarterStageRenderJenkinsfile(
+    context,
+    [source: 'testing.yml.template',
+     target: 'environments/testing.yml']
+  )
+}

e2e-python/Jenkinsfile.template

@@ -0,0 +1,185 @@
+/* generated jenkins file used for building and deploying AWS-infrastructure in projects */
+
+@Library('ods-jenkins-shared-library@@shared_library_ref@') _
+
+node {
+  aws_region = env.AWS_REGION
+  dockerRegistry = env.DOCKER_REGISTRY
+}
+
+odsComponentPipeline(
+  podContainers: [
+      containerTemplate(
+        name: 'jnlp',
+        image: "${dockerRegistry}/ods/jenkins-agent-terraform-2306:@shared_library_ref@",
+        envVars: [
+          envVar(key: 'AWS_REGION', value: aws_region)
+        ],
+        alwaysPullImage: true,
+        args: '${computer.jnlpmac} ${computer.name}'
+      )
+    ],
+  branchToEnvironmentMapping: [
+    '*': 'dev',
+    // 'release/': 'test'
+  ]
+) { context ->
+    getEnvironment(context)
+    addVars2envJsonFile(context)
+    odsComponentStageInfrastructure(context, [cloudProvider: 'AWS'])
+
+    withEnv(["AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}",
+             "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"
+   ])
+   {
+      stage ("AWS Testing Preparation"){
+        generateTerraformOutputsFile()
+      }
+
+      def outputNames = stageGetNamesFromOutputs()
+      def aws_pipelineName = outputNames.aws_codepipeline_name
+      def bitbuckets3_name = outputNames.bitbuckets3_name
+      def results3_name = outputNames.results3_name
+
+      stage ("Publish Bitbucket Code To AWS"){
+        publishBitbucketCodeToAWS(context, bitbuckets3_name)
+      }
+
+      stage ("Run Tests"){
+        awsCodePipelineTrigger(context, aws_pipelineName)
+        awsCodePipelineWaitForExecution(context, aws_pipelineName)
+      }
+
+      stage ("Test Results"){
+        retrieveReportsFromAWS(context, results3_name)
+        archiveArtifacts artifacts: "build/test-results/test/**", allowEmptyArchive: true
+        junit(testResults:'build/test-results/test/*.xml', allowEmptyResults: true)
+        stash(name: "acceptance-test-reports-junit-xml-${context.componentId}-${context.buildNumber}", includes: "build/test-results/test/acceptance*junit.xml", allowEmpty: true)
+        stash(name: "installation-test-reports-junit-xml-${context.componentId}-${context.buildNumber}", includes: "build/test-results/test/installation*junit.xml", allowEmpty: true)
+        stash(name: "integration-test-reports-junit-xml-${context.componentId}-${context.buildNumber}", includes: "build/test-results/test/integration*junit.xml", allowEmpty: true)
+      }
+    }
+
+}
+
+def getEnvironment(def context){
+    sh "echo Get Environment Variables"
+    AWS_ACCESS_KEY_ID = sh(returnStdout: true, script:"oc get secret aws-access-key-id-${context.environment} --namespace ${context.cdProject} --output jsonpath='{.data.secrettext}' | base64 -d")
+    AWS_SECRET_ACCESS_KEY = sh(returnStdout: true, script:"oc get secret aws-secret-access-key-${context.environment} --namespace ${context.cdProject} --output jsonpath='{.data.secrettext}' | base64 -d")
+
+}
+
+
+def generateTerraformOutputsFile() {
+    sh 'terraform output -json > terraform_outputs.json'
+    sh 'cat terraform_outputs.json'
+}
+
+def stageGetNamesFromOutputs() {
+    def outputNames = [:]
+    def terraformOutputJson = readJSON file: 'terraform_outputs.json'
+    //def environmentVarsJson = readJSON file: env.auto.tfvars.json
+
+    outputNames.aws_codepipeline_name = terraformOutputJson.codepipeline_name.value
+    outputNames.bitbuckets3_name = terraformOutputJson.bitbucket_s3bucket_name.value
+    outputNames.results3_name = terraformOutputJson.e2e_results_bucket_name.value
+
+    return outputNames
+}
+
+def awsCodePipelineTrigger(def context, pipelineName) {
+   sh "aws codepipeline start-pipeline-execution --name ${pipelineName}"
+}
+
+
+def awsCodePipelineWaitForExecution(def context, pipelineName) {
+    def pipelineExecutionStatus = ''
+
+    while (true) {
+        pipelineExecutionStatus = ''
+        sleep(time: 40, unit: 'SECONDS')
+        def pipelineState = sh(
+            script: "aws codepipeline get-pipeline-state --name ${pipelineName} --query 'stageStates[*]' --output json",
+            returnStdout: true
+        ).trim()
+
+        def pipelineStages = readJSON(text: pipelineState)
+
+        pipelineStages.each { stage ->
+            def stageName = stage.stageName
+            def stageStatus = stage.latestExecution.status
+            echo "Stage: ${stageName}, Status: ${stageStatus}"
+
+            if (stageStatus == 'InProgress') {
+                pipelineExecutionStatus = 'InProgress'
+                return
+            } else if (stageStatus == 'Failed') {
+                pipelineExecutionStatus = 'Failed'
+                echo "Pipeline execution failed at stage ${stageName}"
+                error("Pipeline execution failed at stage ${stageName}")
+                return
+            }
+        }
+
+        if (pipelineExecutionStatus == 'InProgress') {
+            continue
+        } else if (pipelineExecutionStatus == 'Failed') {
+            echo "Pipeline execution failed at stage ${stageName}"
+            break
+        } else {
+            echo 'Pipeline execution completed successfully.'
+            break
+        }
+    }
+}
+
+
+
+def publishBitbucketCodeToAWS(def context, bitbuckets3_name) {
+    def branch = context.gitBranch
+    def repository = context.componentId
+    zip zipFile: "${repository}-${branch}.zip", archive: false, dir: '.'
+    sh " aws s3 cp ${repository}-${branch}.zip s3://${bitbuckets3_name}/${repository}-${branch}.zip"
+}
+
+def retrieveReportsFromAWS(def context, results3_name) {
+    sh "aws s3 cp s3://${results3_name}/junit/acceptance_GX_junit.xml ./build/test-results/test/acceptance_GX_junit.xml"
+    sh "aws s3 cp s3://${results3_name}/junit/acceptance_pytest_junit.xml ./build/test-results/test/acceptance_pytest_junit.xml"
+    sh "aws s3 cp s3://${results3_name}/junit/installation_pytest_junit.xml ./build/test-results/test/installation_pytest_junit.xml"
+    sh "aws s3 cp s3://${results3_name}/junit/integration_pytest_junit.xml ./build/test-results/test/integration_pytest_junit.xml"
+
+    sh "aws s3 cp s3://${results3_name}/GX_test_results ./build/test-results/test/artifacts/acceptance/acceptance_GX_report --recursive"
+    sh "aws s3 cp s3://${results3_name}/GX_jsons ./build/test-results/test/artifacts/acceptance/GX_jsons --recursive"
+    sh "aws s3 cp s3://${results3_name}/pytest_results/acceptance/acceptance_allure_report_complete.html ./build/test-results/test/artifacts/acceptance/acceptance_pytest_report.html"
+    sh "aws s3 cp s3://${results3_name}/pytest_results/installation/installation_allure_report_complete.html ./build/test-results/test/artifacts/installation/installation_pytest_report.html"
+    sh "aws s3 cp s3://${results3_name}/pytest_results/integration/integration_allure_report_complete.html ./build/test-results/test/artifacts/integration/integration_pytest_report.html"
+
+    sh "ls build/test-results/test"
+    sh "rm build/test-results/test/default.xml"
+}
+
+def addVars2envJsonFile(def context) {
+    echo "Starting addVars2envJsonFile"
+    def environment = context.environment
+    def projectId = context.projectId
+    def branch_name = context.gitBranch
+    def repository = context.componentId
+    def filePath = "./environments/${environment}.json"
+
+    def existingJson = readFile file: filePath
+    def existingData = readJSON text: existingJson
+
+    existingData.environment = environment
+    existingData.projectId = projectId
+    existingData.aws_region = aws_region
+    existingData.repository = repository
+    existingData.branch_name = branch_name
+
+    echo "Environment: ${existingData}"
+
+    def updatedJson = groovy.json.JsonOutput.toJson(existingData)
+    writeFile file: filePath, text: updatedJson
+
+    echo "Finishing addVars2envJsonFile"
+}
+

e2e-python/README.md

@@ -0,0 +1,5 @@
+# e2e-python Quickstarter (e2e-python)
+
+Documentation is located in our [official documentation](https://www.opendevstack.org/ods-documentation/opendevstack/latest/getting-started/index.html)
+
+Please update documentation in the [antora page directory](https://github.com/opendevstack/ods-quickstarters/tree/master/docs/modules/quickstarters/pages)

e2e-python/dev.yml.template

@@ -0,0 +1,7 @@
+region: eu-west-1
+
+credentials:
+  key: @project_id@-cd-aws-access-key-id-dev
+  secret: @project_id@-cd-aws-secret-access-key-dev
+
+account: "<your_aws_account_id>"

e2e-python/files/.devcontainer/devcontainer.json.template

@@ -0,0 +1,3 @@
+{
+  "image": "ghcr.io/nichtraunzer/terrarium:latest"
+}

e2e-python/files/.editorconfig

@@ -0,0 +1,19 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.md]
+trim_trailing_whitespace = false ; trimming trailing whitespace may break Markdown
+
+[Makefile]
+tab_width = 2
+indent_style = tab

e2e-python/files/.gitignore

@@ -0,0 +1,20 @@
+.bundle
+.kitchen
+.terraform
+.terraform.lock.hcl
+.terraform-data.json
+.vscode
+.devcontainer/devcontainer.json
+*.auto.tfvars*
+inspec.lock
+outputs.json
+terraform.tfvars*
+terraform.tfstate*
+tfplan
+vendor
+test/integration/*/files/*.json
+test/integration/*/files/*.yml
+reports/install/*
+!reports/install/.gitkeep
+Pipfile.lock
+.venv