From 63acf23e26e00bf62bf1a3a149d86c3e94cdfa93 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 12 Mar 2023 10:30:06 +0100
Subject: [PATCH 01/33] cleanup from Rm into component pipeline

---
 vars/odsComponentPipeline.groovy | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/vars/odsComponentPipeline.groovy b/vars/odsComponentPipeline.groovy
index ce7d42b56..a7194f3ae 100644
--- a/vars/odsComponentPipeline.groovy
+++ b/vars/odsComponentPipeline.groovy
@@ -1,6 +1,10 @@
 import org.ods.component.Pipeline
 import org.ods.util.Logger
 
+import org.ods.services.ServiceRegistry
+import org.ods.util.ClassLoaderCleaner
+import org.ods.util.UnirestConfig
+
 def call(Map config, Closure body) {
     def debug = env.DEBUG
     if (debug != null) {
@@ -12,7 +16,26 @@ def call(Map config, Closure body) {
     def logger = new Logger(this, config.debug)
     def pipeline = new Pipeline(this, logger)
 
-    pipeline.execute(config, body)
+    try {
+        pipeline.execute(config, body)
+    finally {
+        if (!script.env.MULTI_REPO_BUILD) {
+            logger.debug('-- SHUTTING DOWN Component Pipeline (..) --')
+            logger.resetStopwatch()
+            ServiceRegistry.removeInstance()
+            UnirestConfig.shutdown()
+            try {
+                new ClassLoaderCleaner().clean(logger, processId)
+                // use the jenkins INTERNAL cleanupHeap method - attention NOTHING can happen after this method!
+                logger.debug("forceClean via jenkins internals....")
+                Method cleanupHeap = currentBuild.getRawBuild().getExecution().class.getDeclaredMethod("cleanUpHeap")
+                cleanupHeap.setAccessible(true)
+                cleanupHeap.invoke(currentBuild.getRawBuild().getExecution(), null)
+            } catch (Exception e) {
+                logger.debug("cleanupHeap err: ${e}")
+            }            
+        }
+    }
 }
 
 return this

From b8c3b4b4142f38666ac9bb05ff41117f199f94bd Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 12 Mar 2023 10:34:27 +0100
Subject: [PATCH 02/33] better debug messages

---
 vars/odsComponentPipeline.groovy | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/vars/odsComponentPipeline.groovy b/vars/odsComponentPipeline.groovy
index a7194f3ae..96b0b36fe 100644
--- a/vars/odsComponentPipeline.groovy
+++ b/vars/odsComponentPipeline.groovy
@@ -15,12 +15,17 @@ def call(Map config, Closure body) {
 
     def logger = new Logger(this, config.debug)
     def pipeline = new Pipeline(this, logger)
+    String processId = "${env.JOB_NAME}/${env.BUILD_NUMBER}"
+    UnirestConfig.init()
 
     try {
         pipeline.execute(config, body)
-    finally {
-        if (!script.env.MULTI_REPO_BUILD) {
-            logger.debug('-- SHUTTING DOWN Component Pipeline (..) --')
+     }  finally {
+        if (env.MULTI_REPO_BUILD) {
+            logger.debug('-- in RM mode, shutdown skipped --')
+        }
+        if (!env.MULTI_REPO_BUILD) {
+            logger.warn('-- SHUTTING DOWN Component Pipeline (..) --')
             logger.resetStopwatch()
             ServiceRegistry.removeInstance()
             UnirestConfig.shutdown()

From fae76eb814b8e07142bf01bb1757a6a2c24d6871 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 12 Mar 2023 10:35:49 +0100
Subject: [PATCH 03/33] add imports

---
 vars/odsComponentPipeline.groovy | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/vars/odsComponentPipeline.groovy b/vars/odsComponentPipeline.groovy
index 96b0b36fe..6bdf17470 100644
--- a/vars/odsComponentPipeline.groovy
+++ b/vars/odsComponentPipeline.groovy
@@ -4,6 +4,8 @@ import org.ods.util.Logger
 import org.ods.services.ServiceRegistry
 import org.ods.util.ClassLoaderCleaner
 import org.ods.util.UnirestConfig
+import java.lang.reflect.Method
+import java.nio.file.Paths
 
 def call(Map config, Closure body) {
     def debug = env.DEBUG

From cfab2379dbb0219e1b1bfbeb79e0612763ff016c Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 12 Mar 2023 12:22:21 +0100
Subject: [PATCH 04/33] logger  == null

---
 vars/odsComponentPipeline.groovy | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/vars/odsComponentPipeline.groovy b/vars/odsComponentPipeline.groovy
index 6bdf17470..87ab4b61d 100644
--- a/vars/odsComponentPipeline.groovy
+++ b/vars/odsComponentPipeline.groovy
@@ -18,11 +18,9 @@ def call(Map config, Closure body) {
     def logger = new Logger(this, config.debug)
     def pipeline = new Pipeline(this, logger)
     String processId = "${env.JOB_NAME}/${env.BUILD_NUMBER}"
-    UnirestConfig.init()
-
     try {
         pipeline.execute(config, body)
-     }  finally {
+    }  finally {
         if (env.MULTI_REPO_BUILD) {
             logger.debug('-- in RM mode, shutdown skipped --')
         }
@@ -40,7 +38,8 @@ def call(Map config, Closure body) {
                 cleanupHeap.invoke(currentBuild.getRawBuild().getExecution(), null)
             } catch (Exception e) {
                 logger.debug("cleanupHeap err: ${e}")
-            }            
+            } 
+            logger = null
         }
     }
 }

From 5870b6a00f30b994418998110f4eae22da3368c3 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 12 Mar 2023 18:20:25 +0100
Subject: [PATCH 05/33] cleanup

---
 src/org/ods/component/Pipeline.groovy | 3 ---
 vars/odsComponentPipeline.groovy      | 3 +--
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy
index 92dc221d5..4e7011241 100644
--- a/src/org/ods/component/Pipeline.groovy
+++ b/src/org/ods/component/Pipeline.groovy
@@ -289,9 +289,6 @@ class Pipeline implements Serializable {
                                 "ODS Build Artifacts '${context.componentId}': " +
                                 "\r${JsonOutput.prettyPrint(JsonOutput.toJson(context.getBuildArtifactURIs()))}"
                         )
-                        if (!!!script.env.MULTI_REPO_BUILD) {
-                            cleanUp()
-                        }
                     }
                 }
             }
diff --git a/vars/odsComponentPipeline.groovy b/vars/odsComponentPipeline.groovy
index 87ab4b61d..527f06eee 100644
--- a/vars/odsComponentPipeline.groovy
+++ b/vars/odsComponentPipeline.groovy
@@ -5,7 +5,6 @@ import org.ods.services.ServiceRegistry
 import org.ods.util.ClassLoaderCleaner
 import org.ods.util.UnirestConfig
 import java.lang.reflect.Method
-import java.nio.file.Paths
 
 def call(Map config, Closure body) {
     def debug = env.DEBUG
@@ -38,7 +37,7 @@ def call(Map config, Closure body) {
                 cleanupHeap.invoke(currentBuild.getRawBuild().getExecution(), null)
             } catch (Exception e) {
                 logger.debug("cleanupHeap err: ${e}")
-            } 
+            }
             logger = null
         }
     }

From c8715a5f654aef8a41c0b07509fc4bf43d74a4e2 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Mon, 13 Mar 2023 14:05:18 +0100
Subject: [PATCH 06/33] changelog

---
 CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fee91add2..3bd98e1ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+### Fixed
+- Memory leak fixes for component pipeline ([#857](https://github.com/opendevstack/ods-jenkins-shared-library/issues/857))
+
 ## [4.2.0] - 2023-02-21
 
 ### Added

From da14e18f81044e339150e886d8ee067a33337fde Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 16:54:33 +0100
Subject: [PATCH 07/33] dirty cleanup of skipped builds

---
 src/org/ods/component/Pipeline.groovy      |  3 +++
 src/org/ods/services/JenkinsService.groovy | 16 ++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy
index 4e7011241..8508ffcd7 100644
--- a/src/org/ods/component/Pipeline.groovy
+++ b/src/org/ods/component/Pipeline.groovy
@@ -155,6 +155,9 @@ class Pipeline implements Serializable {
                         }
                     }
 
+                    // check if there is a skipped previous run - if so - delete (to save memory)
+                    jenkinsService.deletePreviousNotBuiltBuild()
+
                     skipCi = isCiSkip()
                     if (skipCi) {
                         script.stage('odsPipeline (ci skip) finished') {
diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index a85e47ed4..4b3b8676c 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -1,6 +1,7 @@
 package org.ods.services
 
 import org.ods.util.ILogger
+import hudson.model.Run
 
 class JenkinsService {
 
@@ -125,4 +126,19 @@ class JenkinsService {
         }
     }
 
+    void deletePreviousNotBuiltBuild () {
+        Run previousBuild = script.currentBuild.getPreviousBuild()?.getRawBuild()
+        if (previousBuild) {
+            logger.debug("Found previous run ${previousBuild.getId()}, ${previousBuild.getNumber()}" +
+                "${previousBuild.getDescription()} res: ${previousBuild.getResult()}")
+            if (previousBuild.getResult() == 'NOT_BUILT') {
+                try {
+                    previousBuild.delete()
+                    logger.debug("deleted build: ${previousBuild.getId()}")
+                } catch (Exception couldNotDelete) {
+                    logger.warn ("Could not delete '${previousBuild.getId()}' - ${couldNotDelete}")
+                }
+            }
+        }
+    }
 }

From 36915cfa9e5fa59db8b3862a6c61c1b9ee1f6442 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 17:04:30 +0100
Subject: [PATCH 08/33] buidl status check

---
 src/org/ods/services/JenkinsService.groovy | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 4b3b8676c..d92ccf8c4 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -129,15 +129,17 @@ class JenkinsService {
     void deletePreviousNotBuiltBuild () {
         Run previousBuild = script.currentBuild.getPreviousBuild()?.getRawBuild()
         if (previousBuild) {
-            logger.debug("Found previous run ${previousBuild.getId()}, ${previousBuild.getNumber()}" +
+            logger.debug("Found previous run: " +
                 "${previousBuild.getDescription()} res: ${previousBuild.getResult()}")
-            if (previousBuild.getResult() == 'NOT_BUILT') {
+            if (previousBuild.getResult().equals('NOT_BUILT')) {
                 try {
                     previousBuild.delete()
                     logger.debug("deleted build: ${previousBuild.getId()}")
                 } catch (Exception couldNotDelete) {
                     logger.warn ("Could not delete '${previousBuild.getId()}' - ${couldNotDelete}")
                 }
+            } else {
+                logger.debug('Skipping deletion of build!')
             }
         }
     }

From eef85a46fef99afa7a02374328c2c4f3112132ce Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 17:08:38 +0100
Subject: [PATCH 09/33] more not built string comparison

---
 src/org/ods/services/JenkinsService.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index d92ccf8c4..3f5a8578c 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -131,7 +131,7 @@ class JenkinsService {
         if (previousBuild) {
             logger.debug("Found previous run: " +
                 "${previousBuild.getDescription()} res: ${previousBuild.getResult()}")
-            if (previousBuild.getResult().equals('NOT_BUILT')) {
+            if (previousBuild.getResult().toString() == 'NOT_BUILT') {
                 try {
                     previousBuild.delete()
                     logger.debug("deleted build: ${previousBuild.getId()}")
@@ -139,7 +139,7 @@ class JenkinsService {
                     logger.warn ("Could not delete '${previousBuild.getId()}' - ${couldNotDelete}")
                 }
             } else {
-                logger.debug('Skipping deletion of build!')
+                logger.debug('Skipping deletion of build, it was not a skip one!')
             }
         }
     }

From 9ddbd890a03050e349550fcb5f0b0a0a3282f452 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 20:13:55 +0100
Subject: [PATCH 10/33] be more specific

---
 src/org/ods/component/Pipeline.groovy      |  4 +++-
 src/org/ods/services/JenkinsService.groovy | 18 ++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy
index 8508ffcd7..4f21add5b 100644
--- a/src/org/ods/component/Pipeline.groovy
+++ b/src/org/ods/component/Pipeline.groovy
@@ -156,7 +156,9 @@ class Pipeline implements Serializable {
                     }
 
                     // check if there is a skipped previous run - if so - delete (to save memory)
-                    jenkinsService.deletePreviousNotBuiltBuild()
+                    if (!MULTI_REPO_BUILD) {
+                        jenkinsService.deletePreviousNotBuiltBuild()
+                    }
 
                     skipCi = isCiSkip()
                     if (skipCi) {
diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 3f5a8578c..9d705b7a2 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -2,6 +2,8 @@ package org.ods.services
 
 import org.ods.util.ILogger
 import hudson.model.Run
+import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper
+import hudson.scm.ChangeLogSet
 
 class JenkinsService {
 
@@ -127,13 +129,25 @@ class JenkinsService {
     }
 
     void deletePreviousNotBuiltBuild () {
-        Run previousBuild = script.currentBuild.getPreviousBuild()?.getRawBuild()
+        RunWrapper previousBuild = script.currentBuild.getPreviousBuild()
         if (previousBuild) {
             logger.debug("Found previous run: " +
                 "${previousBuild.getDescription()} res: ${previousBuild.getResult()}")
             if (previousBuild.getResult().toString() == 'NOT_BUILT') {
+                boolean isCommitRMBased = false
+                if (!previousBuild.getChangeSets()?.isEmpty()) {
+                    ChangeLogSet changes = previousBuild.getChangeSets().get(0)
+                    if (!changes.isEmptySet()) {
+                        ChangeLogSet.Entry change = changes.getItems()[0]
+                        logger.debug("Changlog message: ${change.getMsg()}")
+                        if (change.getMsg().startsWith('ODS: Export OpenShift configuration')) {
+                            isCommitRMBased = true
+                        }
+                    }
+                }
+                if (!isCommitRMBased) return
                 try {
-                    previousBuild.delete()
+                    previousBuild.getRawBuild().delete()
                     logger.debug("deleted build: ${previousBuild.getId()}")
                 } catch (Exception couldNotDelete) {
                     logger.warn ("Could not delete '${previousBuild.getId()}' - ${couldNotDelete}")

From b1d7935ed5ed7b64eb886d84565a98fbf5cf9b5e Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 20:16:08 +0100
Subject: [PATCH 11/33] grab from env

---
 src/org/ods/component/Pipeline.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy
index 4f21add5b..c80b17acb 100644
--- a/src/org/ods/component/Pipeline.groovy
+++ b/src/org/ods/component/Pipeline.groovy
@@ -156,7 +156,7 @@ class Pipeline implements Serializable {
                     }
 
                     // check if there is a skipped previous run - if so - delete (to save memory)
-                    if (!MULTI_REPO_BUILD) {
+                    if (!env.MULTI_REPO_BUILD) {
                         jenkinsService.deletePreviousNotBuiltBuild()
                     }
 

From d89788e267eacf4a1930f7b42428d7172cdaa42c Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 20:17:55 +0100
Subject: [PATCH 12/33] script.env

---
 src/org/ods/component/Pipeline.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy
index c80b17acb..f3ccf20d6 100644
--- a/src/org/ods/component/Pipeline.groovy
+++ b/src/org/ods/component/Pipeline.groovy
@@ -156,7 +156,7 @@ class Pipeline implements Serializable {
                     }
 
                     // check if there is a skipped previous run - if so - delete (to save memory)
-                    if (!env.MULTI_REPO_BUILD) {
+                    if (!script.env.MULTI_REPO_BUILD) {
                         jenkinsService.deletePreviousNotBuiltBuild()
                     }
 

From 07828728c2a65ed96aaad16ec1e4835819ba97eb Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 20:25:45 +0100
Subject: [PATCH 13/33] more debug

---
 src/org/ods/services/JenkinsService.groovy | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 9d705b7a2..37d59dc19 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -142,6 +142,8 @@ class JenkinsService {
                         logger.debug("Changlog message: ${change.getMsg()}")
                         if (change.getMsg().startsWith('ODS: Export OpenShift configuration')) {
                             isCommitRMBased = true
+                        } else {
+                            logger.debug('Found human changelog, previous build will not be deleted')
                         }
                     }
                 }

From 8b4233934a8a5bc8d709f0352e03a83d3ec42155 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 20:40:04 +0100
Subject: [PATCH 14/33] mark for deletion only

---
 src/org/ods/services/JenkinsService.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 37d59dc19..e5799b03d 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -149,7 +149,7 @@ class JenkinsService {
                 }
                 if (!isCommitRMBased) return
                 try {
-                    previousBuild.getRawBuild().delete()
+                    previousBuild.getRawBuild().keepLog​(false)
                     logger.debug("deleted build: ${previousBuild.getId()}")
                 } catch (Exception couldNotDelete) {
                     logger.warn ("Could not delete '${previousBuild.getId()}' - ${couldNotDelete}")

From c1d34f9bf944c808ed63c1f32f27631dd38b0433 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 20:55:34 +0100
Subject: [PATCH 15/33] delete

---
 src/org/ods/services/JenkinsService.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index e5799b03d..37d59dc19 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -149,7 +149,7 @@ class JenkinsService {
                 }
                 if (!isCommitRMBased) return
                 try {
-                    previousBuild.getRawBuild().keepLog​(false)
+                    previousBuild.getRawBuild().delete()
                     logger.debug("deleted build: ${previousBuild.getId()}")
                 } catch (Exception couldNotDelete) {
                     logger.warn ("Could not delete '${previousBuild.getId()}' - ${couldNotDelete}")

From cb576ac614186027e543565c2cd9fbba35951911 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 21:19:26 +0100
Subject: [PATCH 16/33] better error handling

---
 src/org/ods/services/JenkinsService.groovy | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 37d59dc19..b2b5a5bb4 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -131,6 +131,7 @@ class JenkinsService {
     void deletePreviousNotBuiltBuild () {
         RunWrapper previousBuild = script.currentBuild.getPreviousBuild()
         if (previousBuild) {
+            def buildId = "${previousBuild.getId()}"
             logger.debug("Found previous run: " +
                 "${previousBuild.getDescription()} res: ${previousBuild.getResult()}")
             if (previousBuild.getResult().toString() == 'NOT_BUILT') {
@@ -150,12 +151,12 @@ class JenkinsService {
                 if (!isCommitRMBased) return
                 try {
                     previousBuild.getRawBuild().delete()
-                    logger.debug("deleted build: ${previousBuild.getId()}")
-                } catch (Exception couldNotDelete) {
-                    logger.warn ("Could not delete '${previousBuild.getId()}' - ${couldNotDelete}")
+                    logger.debug("deleted build: ${buildId}")
+                } catch (err) {
+                    logger.warn ("Could not delete '${buildId}' - ${err}")
                 }
             } else {
-                logger.debug('Skipping deletion of build, it was not a skip one!')
+                logger.debug('Skipping deletion of build '${buildId}', it was not a skip one!')
             }
         }
     }

From 395fe8aa4a12219369a115feb6749749d210fcf2 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 21:23:17 +0100
Subject: [PATCH 17/33] variable mess

---
 src/org/ods/services/JenkinsService.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index b2b5a5bb4..b649bab62 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -151,12 +151,12 @@ class JenkinsService {
                 if (!isCommitRMBased) return
                 try {
                     previousBuild.getRawBuild().delete()
-                    logger.debug("deleted build: ${buildId}")
+                    logger.debug("Deleted build: ${buildId}")
                 } catch (err) {
                     logger.warn ("Could not delete '${buildId}' - ${err}")
                 }
             } else {
-                logger.debug('Skipping deletion of build '${buildId}', it was not a skip one!')
+                logger.debug("Skipping deletion of build '${buildId}', it was not a skip one!")
             }
         }
     }

From 674bee424dba4ccefbfa059a2193b196ee01128d Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Wed, 15 Mar 2023 21:40:02 +0100
Subject: [PATCH 18/33] better log messages

---
 src/org/ods/services/JenkinsService.groovy | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index b649bab62..48ce3acb7 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -132,8 +132,8 @@ class JenkinsService {
         RunWrapper previousBuild = script.currentBuild.getPreviousBuild()
         if (previousBuild) {
             def buildId = "${previousBuild.getId()}"
-            logger.debug("Found previous run: " +
-                "${previousBuild.getDescription()} res: ${previousBuild.getResult()}")
+            logger.debug("Found previous run: ${buildId}," +
+                "${previousBuild.getDescription()} Result: ${previousBuild.getResult()}")
             if (previousBuild.getResult().toString() == 'NOT_BUILT') {
                 boolean isCommitRMBased = false
                 if (!previousBuild.getChangeSets()?.isEmpty()) {
@@ -151,7 +151,7 @@ class JenkinsService {
                 if (!isCommitRMBased) return
                 try {
                     previousBuild.getRawBuild().delete()
-                    logger.debug("Deleted build: ${buildId}")
+                    logger.info("Deleted build: ${buildId} because it was autogenerated by RM")
                 } catch (err) {
                     logger.warn ("Could not delete '${buildId}' - ${err}")
                 }

From 9388e81993dd9d96ecc30b71957e119a126703a2 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Thu, 16 Mar 2023 15:13:31 +0100
Subject: [PATCH 19/33] add resourceLimit to mro agent

---
 vars/odsOrchestrationPipeline.groovy | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/vars/odsOrchestrationPipeline.groovy b/vars/odsOrchestrationPipeline.groovy
index 8c4f332fc..2067803dc 100644
--- a/vars/odsOrchestrationPipeline.groovy
+++ b/vars/odsOrchestrationPipeline.groovy
@@ -42,6 +42,8 @@ def call(Map config) {
     boolean startAgentEarly = config.get('startOrchestrationAgentOnInit', true)
     def startAgentStage = startAgentEarly ? MROPipelineUtil.PipelinePhases.INIT : null
 
+    resourceLimitMemory = config.get('mroAgentMemoryLimit', '1Gi')
+
     logger.debug("Start agent stage: ${startAgentStage}")
     Project project = new Project(steps, logger)
     def repos = []
@@ -57,7 +59,7 @@ def call(Map config) {
 
             logger.startClocked('pod-template')
             def envs = Project.getBuildEnvironment(steps, debug, versionedDevEnvsEnabled)
-            withPodTemplate(odsImageTag, steps, alwaysPullImage) {
+            withPodTemplate(odsImageTag, steps, alwaysPullImage, resourceLimitMemory) {
                 logger.debugClocked('pod-template')
                 withEnv(envs) {
                     def result
@@ -146,7 +148,8 @@ private void checkOutLocalBranch(GitService git, scm, ILogger logger) {
 }
 
 @SuppressWarnings('GStringExpressionWithinString')
-private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean alwaysPullImage, Closure block) {
+private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean alwaysPullImage, 
+    String mroAgentLimit, Closure block) {
     ILogger logger = ServiceRegistry.instance.get(Logger)
     def dockerRegistry = steps.env.DOCKER_REGISTRY ?: 'docker-registry.default.svc:5000'
     def podLabel = "mro-jenkins-agent-${env.BUILD_NUMBER}"
@@ -165,7 +168,7 @@ private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean always
                 image: "${dockerRegistry}/${odsNamespace}/jenkins-agent-base:${odsImageTag}",
                 workingDir: '/tmp',
                 resourceRequestMemory: '512Mi',
-                resourceLimitMemory: '1Gi',
+                resourceLimitMemory: "${mroAgentLimit}",
                 resourceRequestCpu: '200m',
                 resourceLimitCpu: '1',
                 alwaysPullImage: "${alwaysPullImage}",

From bd59931e019af2d8b4829abfa673fe164ffba2b4 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Thu, 16 Mar 2023 16:26:24 +0100
Subject: [PATCH 20/33] buy stability with time

---
 src/org/ods/orchestration/BuildStage.groovy | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/org/ods/orchestration/BuildStage.groovy b/src/org/ods/orchestration/BuildStage.groovy
index 70c62c65e..95840ca5d 100644
--- a/src/org/ods/orchestration/BuildStage.groovy
+++ b/src/org/ods/orchestration/BuildStage.groovy
@@ -69,19 +69,20 @@ class BuildStage extends Stage {
             }
         }
 
-        Closure generateDocuments = {
+//        Closure generateDocuments = {
             levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.POST_START)
-        }
+//        }
 
         // Execute phase for each repository
-        Closure executeRepos = {
+ //       Closure executeRepos = {
             util.prepareExecutePhaseForReposNamedJob(phase, repos, preExecuteRepo, postExecuteRepo)
                 .each { group ->
                     // FailFast only if not WIP
                     group.failFast = !project.isWorkInProgress
                     script.parallel(group)
                 }
-        }
+ //       }
+
         executeInParallel(executeRepos, generateDocuments)
         levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.PRE_END)
 

From c6cf77fc3db68452c3f8d35475be200cb2792b2b Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Thu, 16 Mar 2023 16:27:39 +0100
Subject: [PATCH 21/33] stabilit fix 2

---
 src/org/ods/orchestration/BuildStage.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/org/ods/orchestration/BuildStage.groovy b/src/org/ods/orchestration/BuildStage.groovy
index 95840ca5d..f3d71ee50 100644
--- a/src/org/ods/orchestration/BuildStage.groovy
+++ b/src/org/ods/orchestration/BuildStage.groovy
@@ -83,7 +83,7 @@ class BuildStage extends Stage {
                 }
  //       }
 
-        executeInParallel(executeRepos, generateDocuments)
+//        executeInParallel(executeRepos, generateDocuments)
         levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.PRE_END)
 
         // in case of WIP we fail AFTER all pieces have been executed - so we can report as many

From 9e2a189d291d284a1b0f0b5de1f24617346ba9ee Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Fri, 17 Mar 2023 08:03:41 +0100
Subject: [PATCH 22/33] cleanup after tests

---
 .../pages/orchestration-pipeline.adoc         | 11 ++++++--
 src/org/ods/orchestration/BuildStage.groovy   | 26 ++++++++-----------
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc b/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc
index 73bf234c2..c7da1aff7 100644
--- a/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc
+++ b/docs/modules/jenkins-shared-library/pages/orchestration-pipeline.adoc
@@ -7,11 +7,11 @@ Load the shared library in your `Jenkinsfile` like this:
 
 [source,groovy]
 ----
-@Library('ods-jenkins-shared-library@3.x') _
+@Library('ods-jenkins-shared-library@4.x') _
 
 odsOrchestrationPipeline(
   debug: true,
-  odsImageTag: '3.x'
+  odsImageTag: '4.x'
 )
 ----
 
@@ -158,6 +158,13 @@ cluster node setup, this may decrease execution performance. In order to re-use
 alwaysPullImage: true
 ----
 
+By default the orchestration pipeline will create a pod based on the jenkins-base-agent image to do much of its work.
+In seldom cases, ususally with a lot of repositories, one may hit an out of memory error on the pod named 'mro-XX'. In this case the below 
+memory limit should be adjusted (defaulting to '1Gi')
+----
+mroAgentMemoryLimit = "1Gi"
+----
+
 == Automated Generation of Compliance Documents
 
 The library automatically generates Lean Validation (LeVA) compliance reports based on data in your Jira project, as well as data generated along the automated build, deploy, test, and release process by the release manager component.
diff --git a/src/org/ods/orchestration/BuildStage.groovy b/src/org/ods/orchestration/BuildStage.groovy
index f3d71ee50..9961d2eec 100644
--- a/src/org/ods/orchestration/BuildStage.groovy
+++ b/src/org/ods/orchestration/BuildStage.groovy
@@ -58,32 +58,28 @@ class BuildStage extends Stage {
                     util.warnBuildIfTestResultsContainFailure(data.tests.unit.testResults)
                 }
 
-                logger.info("levaDocScheduler.run start")
+                logger.debug("levaDocScheduler.run start")
                 levaDocScheduler.run(
                     phase,
                     PipelinePhaseLifecycleStage.POST_EXECUTE_REPO,
                     repo,
                     data
                 )
-                logger.info("levaDocScheduler.run end")
+                logger.debug("levaDocScheduler.run end")
             }
         }
 
-//        Closure generateDocuments = {
-            levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.POST_START)
-//        }
+        // (cut) the reason to NOT go parallel here is a jenkins feature with too many 
+        // parallels causing arraylist$itr serioalouation errors
+        levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.POST_START)
 
-        // Execute phase for each repository
- //       Closure executeRepos = {
-            util.prepareExecutePhaseForReposNamedJob(phase, repos, preExecuteRepo, postExecuteRepo)
-                .each { group ->
-                    // FailFast only if not WIP
-                    group.failFast = !project.isWorkInProgress
-                    script.parallel(group)
-                }
- //       }
+        util.prepareExecutePhaseForReposNamedJob(phase, repos, preExecuteRepo, postExecuteRepo)
+            .each { group ->
+                // FailFast only if not WIP
+                group.failFast = !project.isWorkInProgress
+                script.parallel(group)
+            }
 
-//        executeInParallel(executeRepos, generateDocuments)
         levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.PRE_END)
 
         // in case of WIP we fail AFTER all pieces have been executed - so we can report as many

From 4f8a140b79f51dc6893ee42af31d4b2925bce897 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Fri, 17 Mar 2023 08:08:04 +0100
Subject: [PATCH 23/33] remove plugin dependency with def

---
 src/org/ods/services/JenkinsService.groovy | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 48ce3acb7..17a1815f4 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -2,7 +2,6 @@ package org.ods.services
 
 import org.ods.util.ILogger
 import hudson.model.Run
-import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper
 import hudson.scm.ChangeLogSet
 
 class JenkinsService {
@@ -129,7 +128,7 @@ class JenkinsService {
     }
 
     void deletePreviousNotBuiltBuild () {
-        RunWrapper previousBuild = script.currentBuild.getPreviousBuild()
+        def previousBuild = script.currentBuild.getPreviousBuild()
         if (previousBuild) {
             def buildId = "${previousBuild.getId()}"
             logger.debug("Found previous run: ${buildId}," +

From e0157276640d20138d78512e17cb5ae5b2089736 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Fri, 17 Mar 2023 08:23:00 +0100
Subject: [PATCH 24/33] cleanup and compact cleanupmethod

---
 src/org/ods/component/Pipeline.groovy       | 15 --------
 src/org/ods/orchestration/BuildStage.groovy |  2 +-
 src/org/ods/services/JenkinsService.groovy  | 41 +++++++++------------
 3 files changed, 18 insertions(+), 40 deletions(-)

diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy
index f3ccf20d6..8028f0905 100644
--- a/src/org/ods/component/Pipeline.groovy
+++ b/src/org/ods/component/Pipeline.groovy
@@ -300,21 +300,6 @@ class Pipeline implements Serializable {
         }
     }
 
-    private void cleanUp() {
-        logger.debug('-- SHUTTING DOWN RM (..) --')
-        logger.resetStopwatch()
-        this.script = null
-        this.steps = null
-        this.logger = null
-
-        this.gitService = null
-        this.openShiftService = null
-        this.jenkinsService = null
-        this.bitbucketService = null
-
-        ServiceRegistry.removeInstance()
-    }
-
     def setupForMultiRepoBuild(def config) {
         logger.info '-> Detected multirepo orchestration pipeline build'
         config.localCheckoutEnabled = false
diff --git a/src/org/ods/orchestration/BuildStage.groovy b/src/org/ods/orchestration/BuildStage.groovy
index 9961d2eec..63b4ea60b 100644
--- a/src/org/ods/orchestration/BuildStage.groovy
+++ b/src/org/ods/orchestration/BuildStage.groovy
@@ -69,7 +69,7 @@ class BuildStage extends Stage {
             }
         }
 
-        // (cut) the reason to NOT go parallel here is a jenkins feature with too many 
+        // (cut) the reason to NOT go parallel here is a jenkins feature with too many
         // parallels causing arraylist$itr serioalouation errors
         levaDocScheduler.run(phase, PipelinePhaseLifecycleStage.POST_START)
 
diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 17a1815f4..4b4763000 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -1,7 +1,6 @@
 package org.ods.services
 
 import org.ods.util.ILogger
-import hudson.model.Run
 import hudson.scm.ChangeLogSet
 
 class JenkinsService {
@@ -127,35 +126,29 @@ class JenkinsService {
         }
     }
 
+    // MEM leak saver! delete the previous run, if it was triggered by an RM skip commit
     void deletePreviousNotBuiltBuild () {
         def previousBuild = script.currentBuild.getPreviousBuild()
-        if (previousBuild) {
+        if (previousBuild?.getResult()?.toString() == 'NOT_BUILT') {
             def buildId = "${previousBuild.getId()}"
-            logger.debug("Found previous run: ${buildId}," +
-                "${previousBuild.getDescription()} Result: ${previousBuild.getResult()}")
-            if (previousBuild.getResult().toString() == 'NOT_BUILT') {
-                boolean isCommitRMBased = false
-                if (!previousBuild.getChangeSets()?.isEmpty()) {
-                    ChangeLogSet changes = previousBuild.getChangeSets().get(0)
-                    if (!changes.isEmptySet()) {
-                        ChangeLogSet.Entry change = changes.getItems()[0]
-                        logger.debug("Changlog message: ${change.getMsg()}")
-                        if (change.getMsg().startsWith('ODS: Export OpenShift configuration')) {
-                            isCommitRMBased = true
-                        } else {
-                            logger.debug('Found human changelog, previous build will not be deleted')
+            logger.debug("Found previous CI SKIP run: ${buildId}, ${previousBuild.getDescription()}")
+            // get the change set(s) and look for the first (== last commit and its message)
+            if (!previousBuild.getChangeSets()?.isEmpty()) {
+                ChangeLogSet changes = previousBuild.getChangeSets().get(0)
+                if (!changes.isEmptySet()) {
+                    ChangeLogSet.Entry change = changes.getItems()[0]
+                    logger.debug("Changlog message: ${change.getMsg()}")
+                    if (change.getMsg()?.startsWith('ODS: Export OpenShift configuration')) {
+                        try {
+                            previousBuild.getRawBuild().delete()
+                            logger.info("Deleted (previous) build: ${buildId} because it was autogenerated by RM")
+                        } catch (err) {
+                            logger.warn ("Could not delete build with id: '${buildId}', ${err}")
                         }
+                    } else {
+                        logger.debug("Found human changelog: \n${change.getMsg()}, hence previous build will not be deleted')")
                     }
                 }
-                if (!isCommitRMBased) return
-                try {
-                    previousBuild.getRawBuild().delete()
-                    logger.info("Deleted build: ${buildId} because it was autogenerated by RM")
-                } catch (err) {
-                    logger.warn ("Could not delete '${buildId}' - ${err}")
-                }
-            } else {
-                logger.debug("Skipping deletion of build '${buildId}', it was not a skip one!")
             }
         }
     }

From f0c304f94db6674759c16d10ebaa69b2b28121e0 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Fri, 17 Mar 2023 08:35:48 +0100
Subject: [PATCH 25/33] codenarc

---
 src/org/ods/services/JenkinsService.groovy | 5 +++--
 vars/odsOrchestrationPipeline.groovy       | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 4b4763000..7ab0ac7ff 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -141,12 +141,13 @@ class JenkinsService {
                     if (change.getMsg()?.startsWith('ODS: Export OpenShift configuration')) {
                         try {
                             previousBuild.getRawBuild().delete()
-                            logger.info("Deleted (previous) build: ${buildId} because it was autogenerated by RM")
+                            logger.info("Deleted (previous) build: '${buildId}' because it was autogenerated by RM")
                         } catch (err) {
                             logger.warn ("Could not delete build with id: '${buildId}', ${err}")
                         }
                     } else {
-                        logger.debug("Found human changelog: \n${change.getMsg()}, hence previous build will not be deleted')")
+                        logger.debug("Found human changelog: \n${change.getMsg()}, " +
+                            "hence previous build '${buildId}' will not be deleted")
                     }
                 }
             }
diff --git a/vars/odsOrchestrationPipeline.groovy b/vars/odsOrchestrationPipeline.groovy
index 2067803dc..ea6549c91 100644
--- a/vars/odsOrchestrationPipeline.groovy
+++ b/vars/odsOrchestrationPipeline.groovy
@@ -148,7 +148,7 @@ private void checkOutLocalBranch(GitService git, scm, ILogger logger) {
 }
 
 @SuppressWarnings('GStringExpressionWithinString')
-private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean alwaysPullImage, 
+private withPodTemplate(String odsImageTag, IPipelineSteps steps, boolean alwaysPullImage,
     String mroAgentLimit, Closure block) {
     ILogger logger = ServiceRegistry.instance.get(Logger)
     def dockerRegistry = steps.env.DOCKER_REGISTRY ?: 'docker-registry.default.svc:5000'

From 02136857ad9ef0db31884c9a102e41f38db6e32e Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 19 Mar 2023 08:39:35 +0100
Subject: [PATCH 26/33] go recursive

---
 src/org/ods/component/Pipeline.groovy      |  3 ++-
 src/org/ods/services/JenkinsService.groovy | 14 +++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/org/ods/component/Pipeline.groovy b/src/org/ods/component/Pipeline.groovy
index 8028f0905..6b2918929 100644
--- a/src/org/ods/component/Pipeline.groovy
+++ b/src/org/ods/component/Pipeline.groovy
@@ -157,7 +157,8 @@ class Pipeline implements Serializable {
 
                     // check if there is a skipped previous run - if so - delete (to save memory)
                     if (!script.env.MULTI_REPO_BUILD) {
-                        jenkinsService.deletePreviousNotBuiltBuild()
+                        jenkinsService.deleteNotBuiltBuilds(
+                            script.currentBuild.getPreviousBuild())
                     }
 
                     skipCi = isCiSkip()
diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 7ab0ac7ff..5bee24ef2 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -127,11 +127,13 @@ class JenkinsService {
     }
 
     // MEM leak saver! delete the previous run, if it was triggered by an RM skip commit
-    void deletePreviousNotBuiltBuild () {
-        def previousBuild = script.currentBuild.getPreviousBuild()
+    void deleteNotBuiltBuilds (def previousBuild = null) {
+        if (!previousBuild) {
+            return
+        }
         if (previousBuild?.getResult()?.toString() == 'NOT_BUILT') {
             def buildId = "${previousBuild.getId()}"
-            logger.debug("Found previous CI SKIP run: ${buildId}, ${previousBuild.getDescription()}")
+            logger.debug("Found CI SKIP run: ${buildId}, ${previousBuild.getDescription()}")
             // get the change set(s) and look for the first (== last commit and its message)
             if (!previousBuild.getChangeSets()?.isEmpty()) {
                 ChangeLogSet changes = previousBuild.getChangeSets().get(0)
@@ -141,16 +143,18 @@ class JenkinsService {
                     if (change.getMsg()?.startsWith('ODS: Export OpenShift configuration')) {
                         try {
                             previousBuild.getRawBuild().delete()
-                            logger.info("Deleted (previous) build: '${buildId}' because it was autogenerated by RM")
+                            logger.info("Deleted (CI SKIP) build: '${buildId}' because it was autogenerated by RM")
                         } catch (err) {
                             logger.warn ("Could not delete build with id: '${buildId}', ${err}")
                         }
                     } else {
                         logger.debug("Found human changelog: \n${change.getMsg()}, " +
-                            "hence previous build '${buildId}' will not be deleted")
+                            "hence build '${buildId}' will not be deleted")
                     }
                 }
             }
         }
+        // call this recursively to clean-up all the rm created builds
+        deleteNotBuiltBuilds (previousBuild.getPreviousBuild())
     }
 }

From 88e361c4a5ddaf405dfdabc6d70d20d840005c06 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 19 Mar 2023 08:45:30 +0100
Subject: [PATCH 27/33] fetch early

---
 src/org/ods/services/JenkinsService.groovy | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 5bee24ef2..bd79f41b9 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -130,7 +130,9 @@ class JenkinsService {
     void deleteNotBuiltBuilds (def previousBuild = null) {
         if (!previousBuild) {
             return
-        }
+        }`
+        // we need to do this super early - similar to the id, because once deleted - no access
+        def previousMinusOneBuild = previousBuild.getPreviousBuild()
         if (previousBuild?.getResult()?.toString() == 'NOT_BUILT') {
             def buildId = "${previousBuild.getId()}"
             logger.debug("Found CI SKIP run: ${buildId}, ${previousBuild.getDescription()}")
@@ -155,6 +157,6 @@ class JenkinsService {
             }
         }
         // call this recursively to clean-up all the rm created builds
-        deleteNotBuiltBuilds (previousBuild.getPreviousBuild())
+        deleteNotBuiltBuilds (previousMinusOneBuild)
     }
 }

From 2867416ad78df09332ca30ef41ee413bda64577f Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 19 Mar 2023 08:46:58 +0100
Subject: [PATCH 28/33] typo

---
 src/org/ods/services/JenkinsService.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index bd79f41b9..33d7f4965 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -130,7 +130,7 @@ class JenkinsService {
     void deleteNotBuiltBuilds (def previousBuild = null) {
         if (!previousBuild) {
             return
-        }`
+        }
         // we need to do this super early - similar to the id, because once deleted - no access
         def previousMinusOneBuild = previousBuild.getPreviousBuild()
         if (previousBuild?.getResult()?.toString() == 'NOT_BUILT') {

From f1fedb5d828dc23d0cb67cfd56a28888df9835ac Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 19 Mar 2023 09:09:17 +0100
Subject: [PATCH 29/33] formatting of code

---
 vars/odsComponentPipeline.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vars/odsComponentPipeline.groovy b/vars/odsComponentPipeline.groovy
index 527f06eee..d7c5f5042 100644
--- a/vars/odsComponentPipeline.groovy
+++ b/vars/odsComponentPipeline.groovy
@@ -19,7 +19,7 @@ def call(Map config, Closure body) {
     String processId = "${env.JOB_NAME}/${env.BUILD_NUMBER}"
     try {
         pipeline.execute(config, body)
-    }  finally {
+    } finally {
         if (env.MULTI_REPO_BUILD) {
             logger.debug('-- in RM mode, shutdown skipped --')
         }

From 35b9234c5e2435f5313ef0e496cdbad3be0feb84 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 19 Mar 2023 20:18:04 +0100
Subject: [PATCH 30/33] try to find via SCM revisions

---
 src/org/ods/services/JenkinsService.groovy | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index 33d7f4965..eddd55210 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -154,6 +154,11 @@ class JenkinsService {
                             "hence build '${buildId}' will not be deleted")
                     }
                 }
+            } else {
+                def scmAction = previousBuild?.actions.find { action ->
+                    action instanceof jenkins.scm.api.SCMRevisionAction
+                }
+                logger.debug("Found SCM: ${scmAction?.revision}")
             }
         }
         // call this recursively to clean-up all the rm created builds

From 5be16172ab13509e840d1c613d4fd15f048665d5 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 19 Mar 2023 20:22:35 +0100
Subject: [PATCH 31/33] old dead code

---
 src/org/ods/services/JenkinsService.groovy | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index eddd55210..c438a446b 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -155,10 +155,8 @@ class JenkinsService {
                     }
                 }
             } else {
-                def scmAction = previousBuild?.actions.find { action ->
-                    action instanceof jenkins.scm.api.SCMRevisionAction
-                }
-                logger.debug("Found SCM: ${scmAction?.revision}")
+                def causes = previousBuild?.getRawBuild().getCauses()
+                logger.debug("Found SCM: ${causes}")
             }
         }
         // call this recursively to clean-up all the rm created builds

From 7430076a52e3c69bdc8a12852861da8547061989 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Sun, 19 Mar 2023 22:34:36 +0100
Subject: [PATCH 32/33] dump old env for debug

---
 src/org/ods/services/JenkinsService.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index c438a446b..a874e0684 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -132,6 +132,7 @@ class JenkinsService {
             return
         }
         // we need to do this super early - similar to the id, because once deleted - no access
+        // option2 -reset the result to SUCCESS
         def previousMinusOneBuild = previousBuild.getPreviousBuild()
         if (previousBuild?.getResult()?.toString() == 'NOT_BUILT') {
             def buildId = "${previousBuild.getId()}"
@@ -155,8 +156,7 @@ class JenkinsService {
                     }
                 }
             } else {
-                def causes = previousBuild?.getRawBuild().getCauses()
-                logger.debug("Found SCM: ${causes}")
+                logger.debug("Environment: ${previousBuild.getBuildVariables()}")
             }
         }
         // call this recursively to clean-up all the rm created builds

From b46c8cd30224f0fd7bec4e4c15a385e78d4a1fe5 Mon Sep 17 00:00:00 2001
From: clemensutschig <clemens.utschig-utschig@boehringer-ingelheim.com>
Date: Mon, 20 Mar 2023 09:12:23 +0100
Subject: [PATCH 33/33] add also case for deployment state with openshift dir

---
 src/org/ods/services/JenkinsService.groovy | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/org/ods/services/JenkinsService.groovy b/src/org/ods/services/JenkinsService.groovy
index a874e0684..e70d7c7f1 100644
--- a/src/org/ods/services/JenkinsService.groovy
+++ b/src/org/ods/services/JenkinsService.groovy
@@ -143,7 +143,8 @@ class JenkinsService {
                 if (!changes.isEmptySet()) {
                     ChangeLogSet.Entry change = changes.getItems()[0]
                     logger.debug("Changlog message: ${change.getMsg()}")
-                    if (change.getMsg()?.startsWith('ODS: Export OpenShift configuration')) {
+                    if (change.getMsg()?.startsWith('ODS: Export OpenShift configuration') ||
+                        change.getMsg()?.startsWith('ODS: Export Openshift deployment state')) {
                         try {
                             previousBuild.getRawBuild().delete()
                             logger.info("Deleted (CI SKIP) build: '${buildId}' because it was autogenerated by RM")
@@ -155,8 +156,6 @@ class JenkinsService {
                             "hence build '${buildId}' will not be deleted")
                     }
                 }
-            } else {
-                logger.debug("Environment: ${previousBuild.getBuildVariables()}")
             }
         }
         // call this recursively to clean-up all the rm created builds