diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index b9c65bc912f12f..71dd85001328d5 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -9,11 +9,23 @@ jobs:
   cache:
     uses: ./.github/workflows/cache-shared-deps.yml
 
-  test:
+  test-common:
     needs:
     - cache
 
-    uses: ./.github/workflows/test-all.yml
+    uses: ./.github/workflows/tests/test-common.yml
+    with:
+      repo: core
+
+      # Options
+      standard: true
+    secrets: inherit
+      
+  test-integrations:
+    needs:
+    - cache
+
+    uses: ./.github/workflows/tests/test-all.yml
     with:
       repo: core
 
@@ -23,7 +35,8 @@ jobs:
 
   publish-test-results:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
     concurrency:
       group: test-results
@@ -33,7 +46,8 @@ jobs:
 
   submit-traces:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
 
     uses: ./.github/workflows/submit-traces.yml
diff --git a/.github/workflows/nightly-base-package.yml b/.github/workflows/nightly-base-package.yml
index 75ef7a8eb58c40..7249800734c517 100644
--- a/.github/workflows/nightly-base-package.yml
+++ b/.github/workflows/nightly-base-package.yml
@@ -6,8 +6,19 @@ on:
   - cron: "0 5 * * *"
 
 jobs:
-  test:
-    uses: ./.github/workflows/test-all.yml
+  test-common:
+    uses: ./.github/workflows/tests/test-common.yml
+    with:
+      repo: core
+
+      # Options
+      standard: true
+      minimum-base-package: true
+      test-py2: true
+    secrets: inherit
+  
+  test-integrations:
+    uses: ./.github/workflows/tests/test-all.yml
     with:
       repo: core
 
@@ -19,7 +30,8 @@ jobs:
 
   submit-traces:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
 
     uses: ./.github/workflows/submit-traces.yml
diff --git a/.github/workflows/nightly-py2.yml b/.github/workflows/nightly-py2.yml
index 563d71890cf3a1..bc871781c04384 100644
--- a/.github/workflows/nightly-py2.yml
+++ b/.github/workflows/nightly-py2.yml
@@ -6,8 +6,19 @@ on:
   - cron: "0 4 * * *"
 
 jobs:
-  test:
-    uses: ./.github/workflows/test-all.yml
+  test-common:
+    uses: ./.github/workflows/tests/test-common.yml
+    with:
+      repo: core
+
+      # Options
+      standard: true
+      test-py2: true
+      test-py3: false
+    secrets: inherit
+  
+  test-integrations:
+    uses: ./.github/workflows/tests/test-all.yml
     with:
       repo: core
 
@@ -19,7 +30,8 @@ jobs:
 
   submit-traces:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
 
     uses: ./.github/workflows/submit-traces.yml
diff --git a/.github/workflows/pr-all.yml b/.github/workflows/pr-all.yml
index dbf2641211ff81..1af8ed3de27498 100644
--- a/.github/workflows/pr-all.yml
+++ b/.github/workflows/pr-all.yml
@@ -15,8 +15,18 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
-    uses: ./.github/workflows/test-all.yml
+  test-common:
+    uses: ./.github/workflows/tests/test-common.yml
+    with:
+      repo: core
+
+      # Options
+      standard: true
+      test-py2: true
+    secrets: inherit
+
+  test-integrations:
+    uses: ./.github/workflows/tests/test-all.yml
     with:
       repo: core
 
@@ -27,14 +37,16 @@ jobs:
 
   save-event:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
 
     uses: ./.github/workflows/save-event.yml
 
   submit-traces:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
 
     uses: ./.github/workflows/submit-traces.yml
diff --git a/.github/workflows/test-agent.yml b/.github/workflows/test-agent.yml
index 4cc4a86d90d065..2b8313f59f5aa1 100644
--- a/.github/workflows/test-agent.yml
+++ b/.github/workflows/test-agent.yml
@@ -35,8 +35,23 @@ on:
         type: string
 
 jobs:
-  test:
-    uses: ./.github/workflows/test-all.yml
+  test-common:
+    uses: ./.github/workflows/tests/test-common.yml
+    with:
+      repo: core
+
+      # Options
+      standard: true
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+    secrets: inherit
+  
+  test-integrations:
+    uses: ./.github/workflows/tests/test-all.yml
     with:
       repo: core
 
@@ -52,7 +67,8 @@ jobs:
 
   submit-traces:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
 
     uses: ./.github/workflows/submit-traces.yml
diff --git a/.github/workflows/test-all.yml b/.github/workflows/tests/test-all.yml
similarity index 95%
rename from .github/workflows/test-all.yml
rename to .github/workflows/tests/test-all.yml
index 5ca8f198d8ab8a..ceb69b00be7fee 100644
--- a/.github/workflows/test-all.yml
+++ b/.github/workflows/tests/test-all.yml
@@ -1,4 +1,4 @@
-name: Test all
+name: Test all integrations
 
 on:
   workflow_call:
@@ -49,158 +49,6 @@ on:
         type: string
 
 jobs:
-  jd316aba:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: ddev on Linux
-      target: ddev
-      platform: linux
-      runner: '["ubuntu-22.04"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
-  j6712d43:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: ddev on Windows
-      target: ddev
-      platform: windows
-      runner: '["windows-2022"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
-  jb232c8c:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: Datadog Checks Base on Linux
-      target: datadog_checks_base
-      platform: linux
-      runner: '["ubuntu-22.04"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
-  jda96080:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: Datadog Checks Base on Windows
-      target: datadog_checks_base
-      platform: windows
-      runner: '["windows-2022"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
-  j77217e9:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: Datadog Checks Dev on Linux
-      target: datadog_checks_dev
-      platform: linux
-      runner: '["ubuntu-22.04"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
-  j1176210:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: Datadog Checks Dev on Windows
-      target: datadog_checks_dev
-      platform: windows
-      runner: '["windows-2022"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
-  j8cdb65b:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: Datadog Checks Downloader on Linux
-      target: datadog_checks_downloader
-      platform: linux
-      runner: '["ubuntu-22.04"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
-  j8c73c41:
-    uses: ./.github/workflows/test-target.yml
-    with:
-      job-name: Datadog Checks Downloader on Windows
-      target: datadog_checks_downloader
-      platform: windows
-      runner: '["windows-2022"]'
-      repo: "${{ inputs.repo }}"
-      python-version: "${{ inputs.python-version }}"
-      standard: ${{ inputs.standard }}
-      latest: ${{ inputs.latest }}
-      agent-image: "${{ inputs.agent-image }}"
-      agent-image-py2: "${{ inputs.agent-image-py2 }}"
-      agent-image-windows: "${{ inputs.agent-image-windows }}"
-      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
-      test-py2: ${{ inputs.test-py2 }}
-      test-py3: ${{ inputs.test-py3 }}
-      minimum-base-package: ${{ inputs.minimum-base-package }}
-    secrets: inherit
   jd66e148:
     uses: ./.github/workflows/test-target.yml
     with:
diff --git a/.github/workflows/tests/test-common.yml b/.github/workflows/tests/test-common.yml
new file mode 100644
index 00000000000000..9ace82036037bd
--- /dev/null
+++ b/.github/workflows/tests/test-common.yml
@@ -0,0 +1,203 @@
+name: Test areas common to integrations
+
+on:
+  workflow_call:
+    inputs:
+      repo:
+        required: true
+        type: string
+
+      python-version:
+        required: false
+        default: ""
+        type: string
+      standard:
+        required: false
+        default: false
+        type: boolean
+      latest:
+        required: false
+        default: false
+        type: boolean
+      minimum-base-package:
+        required: false
+        default: false
+        type: boolean
+      test-py2:
+        required: false
+        default: false
+        type: boolean
+      test-py3:
+        required: false
+        default: true
+        type: boolean
+      agent-image:
+        required: false
+        default: ""
+        type: string
+      agent-image-py2:
+        required: false
+        default: ""
+        type: string
+      agent-image-windows:
+        required: false
+        default: ""
+        type: string
+      agent-image-windows-py2:
+        required: false
+        default: ""
+        type: string
+
+jobs:
+  jd316aba:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: ddev on Linux
+      target: ddev
+      platform: linux
+      runner: '["ubuntu-22.04"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
+  j6712d43:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: ddev on Windows
+      target: ddev
+      platform: windows
+      runner: '["windows-2022"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
+  jb232c8c:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: Datadog Checks Base on Linux
+      target: datadog_checks_base
+      platform: linux
+      runner: '["ubuntu-22.04"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
+  jda96080:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: Datadog Checks Base on Windows
+      target: datadog_checks_base
+      platform: windows
+      runner: '["windows-2022"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
+  j77217e9:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: Datadog Checks Dev on Linux
+      target: datadog_checks_dev
+      platform: linux
+      runner: '["ubuntu-22.04"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
+  j1176210:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: Datadog Checks Dev on Windows
+      target: datadog_checks_dev
+      platform: windows
+      runner: '["windows-2022"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
+  j8cdb65b:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: Datadog Checks Downloader on Linux
+      target: datadog_checks_downloader
+      platform: linux
+      runner: '["ubuntu-22.04"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
+  j8c73c41:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: Datadog Checks Downloader on Windows
+      target: datadog_checks_downloader
+      platform: windows
+      runner: '["windows-2022"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      minimum-base-package: ${{ inputs.minimum-base-package }}
+    secrets: inherit
diff --git a/.github/workflows/weekly-latest.yml b/.github/workflows/weekly-latest.yml
index 697fec9bbc83e1..d7d7fbb23dd329 100644
--- a/.github/workflows/weekly-latest.yml
+++ b/.github/workflows/weekly-latest.yml
@@ -6,8 +6,17 @@ on:
   - cron: "0 5 * * 1"
 
 jobs:
-  test:
-    uses: ./.github/workflows/test-all.yml
+  test-common:
+    uses: ./.github/workflows/tests/test-common.yml
+    with:
+      repo: core
+
+      # Options
+      latest: true
+    secrets: inherit
+  
+  test-integrations:
+    uses: ./.github/workflows/tests/test-all.yml
     with:
       repo: core
 
@@ -17,7 +26,8 @@ jobs:
 
   submit-traces:
     needs:
-    - test
+    - test-common
+    - test-integrations
     if: success() || failure()
 
     uses: ./.github/workflows/submit-traces.yml
diff --git a/ddev/changelog.d/17602.fixed b/ddev/changelog.d/17602.fixed
new file mode 100644
index 00000000000000..b35348126ca50c
--- /dev/null
+++ b/ddev/changelog.d/17602.fixed
@@ -0,0 +1 @@
+Extract ddev, datadog_checks_base and datadog_checks_dev to separate workflows
diff --git a/ddev/src/ddev/cli/validate/ci.py b/ddev/src/ddev/cli/validate/ci.py
index d883b477a4a648..bdf9eea1cc0a2e 100644
--- a/ddev/src/ddev/cli/validate/ci.py
+++ b/ddev/src/ddev/cli/validate/ci.py
@@ -60,8 +60,6 @@ def ci(app: Application, sync: bool):
         if app.repo.name == 'core'
         else 'DataDog/integrations-core/.github/workflows/test-target.yml@master'
     )
-    jobs_workflow_path = app.repo.path / '.github' / 'workflows' / 'test-all.yml'
-    original_jobs_workflow = jobs_workflow_path.read_text() if jobs_workflow_path.is_file() else ''
 
     jobs = {}
     for data in construct_job_matrix(app.repo.path, get_all_targets(app.repo.path)):
@@ -120,14 +118,49 @@ def ci(app: Application, sync: bool):
     ):
         jobs_component = jobs_component.replace(f'${{{{ inputs.{field} }}}}', f'"${{{{ inputs.{field} }}}}"')
 
-    manual_component = original_jobs_workflow.split('jobs:')[0].strip()
-    expected_jobs_workflow = f'{manual_component}\n\n{jobs_component}'
+    # TODO: Figure out a way to handle test-common.yml
+    # common_jobs_workflow_path = app.repo.path / '.github' / 'workflows' / 'test-common.yml'
+    # original_common_jobs_workflow = common_jobs_workflow_path.read_text() \
+    #   if common_jobs_workflow_path.is_file() else ''
+    jobs_workflow_files = [
+        'test-all.yml',
+        'test-common.yml',
+    ]
+    workflow_job_data = {}
+    derived_jobs = {}
+    missing_jobs = {}
+    invalid_jobs = {}
+    for jobs_workflow_file in jobs_workflow_files:
+        jobs_workflow_path = app.repo.path / '.github' / 'workflows' / 'tests' / jobs_workflow_file
+        with jobs_workflow_path.open() as workflow_file:
+            original_jobs_workflow = yaml.safe_load(workflow_file)
+            workflow_job_data[jobs_workflow_file] = {
+                'name': original_jobs_workflow.get('name', ''),
+                'triggers': original_jobs_workflow.get('on', {}),
+                'jobs': original_jobs_workflow.get('jobs', {}),
+            }
+            derived_jobs[jobs_workflow_file] = original_jobs_workflow.get('jobs', {})
+
+    for job, job_data in jobs.items():
+        job_found = False
+        for workflow_file_name, derived_jobs_in_workflow in derived_jobs.items():
+            if job in derived_jobs_in_workflow:
+                job_found = True
+                if derived_jobs_in_workflow[job] != job_data:
+                    print(f'Job {job} is invalid in {workflow_file_name}')
+                    print(f'Expected: {job_data}')
+                    print(f'Actual: {derived_jobs_in_workflow[job]}')
+                    invalid_jobs[job] = derived_jobs_in_workflow[job]
+                break
+        if not job_found:
+            print(
+                f'Job {job} with data {job_data} is missing, \
+please add it to one of the relevant workflow files {jobs_workflow_files}'
+            )
+            missing_jobs[job] = job_data
 
-    if original_jobs_workflow != expected_jobs_workflow:
-        if sync:
-            jobs_workflow_path.write_text(expected_jobs_workflow)
-        else:
-            app.abort('CI configuration is not in sync, try again with the `--sync` flag')
+    if missing_jobs or invalid_jobs:
+        app.abort('CI configuration is not in sync, please correct it manually based on output logs')
 
     validation_tracker = app.create_validation_tracker('CI configuration validation')
     error_message = ''
diff --git a/docs/developer/meta/ci/testing.md b/docs/developer/meta/ci/testing.md
index 3377cdfb584064..b505c1624b0d26 100644
--- a/docs/developer/meta/ci/testing.md
+++ b/docs/developer/meta/ci/testing.md
@@ -144,7 +144,7 @@ exclude = true
 
 ## Target enumeration
 
-The [list](https://github.com/DataDog/integrations-core/blob/master/.github/workflows/test-all.yml) of all jobs is [generated](validation.md#ci-configuration) as the `/.github/workflows/test-all.yml` file.
+The [list](https://github.com/DataDog/integrations-core/blob/master/.github/workflows/tests/test-all.yml) of all jobs is [generated](validation.md#ci-configuration) as the `/.github/workflows/tests/test-all.yml` file.
 
 This reusable workflow is called by workflows that need to test everything.