From d2e48c0aa548aa7e4127515e4606a80753b67845 Mon Sep 17 00:00:00 2001 From: brais <26645694+braisvq1996@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:43:29 +0100 Subject: [PATCH] Align master branch commits with 4.x (#986) * 4.x Memory leak cleanup code - for component pipeline (#980) --- CHANGELOG.md | 3 +- .../pages/orchestration-pipeline.adoc | 11 ++++-- src/org/ods/component/Pipeline.groovy | 24 ++++--------- src/org/ods/orchestration/BuildStage.groovy | 27 +++++++------- src/org/ods/services/JenkinsService.groovy | 36 +++++++++++++++++++ vars/odsComponentPipeline.groovy | 32 +++++++++++++++-- vars/odsOrchestrationPipeline.groovy | 9 +++-- 7 files changed, 101 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24889762c..985826bfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # Changelog ## Unreleased -- Fix Component fails to deploy with more than one DeploymentConfig([#981](https://github.com/opendevstack/ods-jenkins-shared-library/issues/981)) ### Fixed - Change default registry to match Openshift 4 ([#983] (https://github.com/opendevstack/ods-jenkins-shared-library/issues/983)) +- Memory leak fixes for component pipeline ([#857](https://github.com/opendevstack/ods-jenkins-shared-library/issues/857)) +- Fix Component fails to deploy with more than one DeploymentConfig([#981](https://github.com/opendevstack/ods-jenkins-shared-library/issues/981)) ## [4.2.0] - 2023-02-21 diff --git a/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc b/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc index 73bf234c2..c7da1aff7 100644 --- a/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc +++ b/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc @@ -7,11 +7,11 @@ Load the shared library in your `Jenkinsfile` like this: [source,groovy] ---- -@Library('ods-jenkins-shared-library@3.x') _ +@Library('ods-jenkins-shared-library@4.x') _ odsOrchestrationPipeline( debug: true, - odsImageTag: '3.x' + odsImageTag: '4.x' ) ---- @@ -158,6 +158,13 @@ cluster node setup, this may decrease execution performance. In order to re-use alwaysPullImage: true ---- +By default the orchestration pipeline will create a pod based on the jenkins-base-agent image to do much of its work. +In seldom cases, ususally with a lot of repositories, one may hit an out of memory error on the pod named 'mro-XX'. In this case the below +memory limit should be adjusted (defaulting to '1Gi') +---- +mroAgentMemoryLimit = "1Gi" +---- + == Automated Generation of Compliance Documents The library automatically generates Lean Validation (LeVA) compliance reports based on data in your Jira project, as well as data generated along the automated build, deploy, test, and release process by the release manager component. diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy index c0d042e7c..8a84737e1 100644 --- a/src/org/ods/component/Pipeline.groovy +++ b/src/org/ods/component/Pipeline.groovy @@ -155,6 +155,12 @@ class Pipeline implements Serializable { } } + // check if there is a skipped previous run - if so - delete (to save memory) + if (!script.env.MULTI_REPO_BUILD) { + jenkinsService.deleteNotBuiltBuilds( + script.currentBuild.getPreviousBuild()) + } + skipCi = isCiSkip() if (skipCi) { script.stage('odsPipeline (ci skip) finished') { @@ -289,30 +295,12 @@ class Pipeline implements Serializable { "ODS Build Artifacts '${context.componentId}': " + "\r${JsonOutput.prettyPrint(JsonOutput.toJson(context.getBuildArtifactURIs()))}" ) - if (!!!script.env.MULTI_REPO_BUILD) { - cleanUp() - } } } } } } - private void cleanUp() { - logger.debug('-- SHUTTING DOWN RM (..) --') - logger.resetStopwatch() - this.script = null - this.steps = null - this.logger = null - - this.gitService = null - this.openShiftService = null - this.jenkinsService = null - this.bitbucketService = null - - ServiceRegistry.removeInstance() - } - def setupForMultiRepoBuild(def config) { logger.info '-> Detected multirepo orchestration pipeline build' config.localCheckoutEnabled = false diff --git a/src/org/ods/orchestration/BuildStage.groovy b/src/org/ods/orchestration/BuildStage.groovy index 70c62c65e..63b4ea60b 100644 --- a/src/org/ods/orchestration/BuildStage.groovy +++ b/src/org/ods/orchestration/BuildStage.groovy @@ -58,31 +58,28 @@ class BuildStage extends Stage { util.warnBuildIfTestResultsContainFailure(data.tests.unit.testResults) } - logger.info("levaDocScheduler.run start") + logger.debug("levaDocScheduler.run start") levaDocScheduler.run( phase, PipelinePhaseLifecycleStage.POST_EXECUTE_REPO, repo, data ) - logger.info("levaDocScheduler.run end") + logger.debug("levaDocScheduler.run end") } } - Closure generateDocuments = { - levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.POST_START) - } + // (cut) the reason to NOT go parallel here is a jenkins feature with too many + // parallels causing arraylist$itr serioalouation errors + levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.POST_START) + + util.prepareExecutePhaseForReposNamedJob(phase, repos, preExecuteRepo, postExecuteRepo) + .each { group -> + // FailFast only if not WIP + group.failFast = !project.isWorkInProgress + script.parallel(group) + } - // Execute phase for each repository - Closure executeRepos = { - util.prepareExecutePhaseForReposNamedJob(phase, repos, preExecuteRepo, postExecuteRepo) - .each { group -> - // FailFast only if not WIP - group.failFast = !project.isWorkInProgress - script.parallel(group) - } - } - executeInParallel(executeRepos, generateDocuments) levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.PRE_END) // in case of WIP we fail AFTER all pieces have been executed - so we can report as many diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy index a85e47ed4..e70d7c7f1 100644 --- a/src/org/ods/services/JenkinsService.groovy +++ b/src/org/ods/services/JenkinsService.groovy @@ -1,6 +1,7 @@ package org.ods.services import org.ods.util.ILogger +import hudson.scm.ChangeLogSet class JenkinsService { @@ -125,4 +126,39 @@ class JenkinsService { } } + // MEM leak saver! delete the previous run, if it was triggered by an RM skip commit + void deleteNotBuiltBuilds (def previousBuild = null) { + if (!previousBuild) { + return + } + // we need to do this super early - similar to the id, because once deleted - no access + // option2 -reset the result to SUCCESS + def previousMinusOneBuild = previousBuild.getPreviousBuild() + if (previousBuild?.getResult()?.toString() == 'NOT_BUILT') { + def buildId = "${previousBuild.getId()}" + logger.debug("Found CI SKIP run: ${buildId}, ${previousBuild.getDescription()}") + // get the change set(s) and look for the first (== last commit and its message) + if (!previousBuild.getChangeSets()?.isEmpty()) { + ChangeLogSet changes = previousBuild.getChangeSets().get(0) + if (!changes.isEmptySet()) { + ChangeLogSet.Entry change = changes.getItems()[0] + logger.debug("Changlog message: ${change.getMsg()}") + if (change.getMsg()?.startsWith('ODS: Export OpenShift configuration') || + change.getMsg()?.startsWith('ODS: Export Openshift deployment state')) { + try { + previousBuild.getRawBuild().delete() + logger.info("Deleted (CI SKIP) build: '${buildId}' because it was autogenerated by RM") + } catch (err) { + logger.warn ("Could not delete build with id: '${buildId}', ${err}") + } + } else { + logger.debug("Found human changelog: \n${change.getMsg()}, " + + "hence build '${buildId}' will not be deleted") + } + } + } + } + // call this recursively to clean-up all the rm created builds + deleteNotBuiltBuilds (previousMinusOneBuild) + } } diff --git a/vars/odsComponentPipeline.groovy b/vars/odsComponentPipeline.groovy index ce7d42b56..d7c5f5042 100644 --- a/vars/odsComponentPipeline.groovy +++ b/vars/odsComponentPipeline.groovy @@ -1,6 +1,11 @@ import org.ods.component.Pipeline import org.ods.util.Logger +import org.ods.services.ServiceRegistry +import org.ods.util.ClassLoaderCleaner +import org.ods.util.UnirestConfig +import java.lang.reflect.Method + def call(Map config, Closure body) { def debug = env.DEBUG if (debug != null) { @@ -11,8 +16,31 @@ def call(Map config, Closure body) { def logger = new Logger(this, config.debug) def pipeline = new Pipeline(this, logger) - - pipeline.execute(config, body) + String processId = "${env.JOB_NAME}/${env.BUILD_NUMBER}" + try { + pipeline.execute(config, body) + } finally { + if (env.MULTI_REPO_BUILD) { + logger.debug('-- in RM mode, shutdown skipped --') + } + if (!env.MULTI_REPO_BUILD) { + logger.warn('-- SHUTTING DOWN Component Pipeline (..) --') + logger.resetStopwatch() + ServiceRegistry.removeInstance() + UnirestConfig.shutdown() + try { + new ClassLoaderCleaner().clean(logger, processId) + // use the jenkins INTERNAL cleanupHeap method - attention NOTHING can happen after this method! + logger.debug("forceClean via jenkins internals....") + Method cleanupHeap = currentBuild.getRawBuild().getExecution().class.getDeclaredMethod("cleanUpHeap") + cleanupHeap.setAccessible(true) + cleanupHeap.invoke(currentBuild.getRawBuild().getExecution(), null) + } catch (Exception e) { + logger.debug("cleanupHeap err: ${e}") + } + logger = null + } + } } return this diff --git a/vars/odsOrchestrationPipeline.groovy b/vars/odsOrchestrationPipeline.groovy index 556e1db9c..9ba6ee180 100644 --- a/vars/odsOrchestrationPipeline.groovy +++ b/vars/odsOrchestrationPipeline.groovy @@ -42,6 +42,8 @@ def call(Map config) { boolean startAgentEarly = config.get('startOrchestrationAgentOnInit', true) def startAgentStage = startAgentEarly ? MROPipelineUtil.PipelinePhases.INIT : null + resourceLimitMemory = config.get('mroAgentMemoryLimit', '1Gi') + logger.debug("Start agent stage: ${startAgentStage}") Project project = new Project(steps, logger) def repos = [] @@ -57,7 +59,7 @@ def call(Map config) { logger.startClocked('pod-template') def envs = Project.getBuildEnvironment(steps, debug, versionedDevEnvsEnabled) - withPodTemplate(odsImageTag, steps, alwaysPullImage) { + withPodTemplate(odsImageTag, steps, alwaysPullImage, resourceLimitMemory) { logger.debugClocked('pod-template') withEnv(envs) { def result @@ -146,7 +148,8 @@ private void checkOutLocalBranch(GitService git, scm, ILogger logger) { } @SuppressWarnings('GStringExpressionWithinString') -private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean alwaysPullImage, Closure block) { +private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean alwaysPullImage, + String mroAgentLimit, Closure block) { ILogger logger = ServiceRegistry.instance.get(Logger) def dockerRegistry = steps.env.DOCKER_REGISTRY ?: 'image-registry.openshift-image-registry.svc:5000' def podLabel = "mro-jenkins-agent-${env.BUILD_NUMBER}" @@ -165,7 +168,7 @@ private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean always image: "${dockerRegistry}/${odsNamespace}/jenkins-agent-base:${odsImageTag}", workingDir: '/tmp', resourceRequestMemory: '512Mi', - resourceLimitMemory: '1Gi', + resourceLimitMemory: "${mroAgentLimit}", resourceRequestCpu: '200m', resourceLimitCpu: '1', alwaysPullImage: "${alwaysPullImage}",