Merge pull request #22 from advanced-security/jsinglet/repo-library-i…

…ssue

adding new option for default codescanning config

.github/workflows/internal-pr-bundle-integration-test-cpp.yml

@@ -66,13 +66,12 @@ jobs:
       with:
         languages: ${{ matrix.language }}
         queries: security-extended
-        source-root: integration-tests/cpp/src/  # Path containing the example application 
         tools: ${{ env.QLT_CODEQL_BUNDLE_PATH }}
 
     - name: Autobuild
       uses: github/codeql-action/autobuild@v2
       with:
-        working-directory: integration-tests/cpp/src/  # Path containing the example application 
+        working-directory: example/integration-tests/cpp/src/  # Path containing the example application 
 
     - name: Perform CodeQL Analysis
       id: analysis
@@ -105,4 +104,4 @@ jobs:
       shell: bash
       run: |
         # Compare the expected vs the actual 
-        qlt bundle run validate-integration-tests --expected integration-tests/cpp/expected.sarif --actual ${{ steps.analysis.outputs.sarif-output }}/cpp.sarif        
+        qlt bundle run validate-integration-tests --expected example/integration-tests/cpp/expected.sarif --actual ${{ steps.analysis.outputs.sarif-output }}/cpp.sarif        

.github/workflows/internal-validate-workflow-files.yml

@@ -47,6 +47,15 @@ jobs:
             exit 1
           fi
 
+      - name: Generate Workflow Files (Bundle Feature)
+        shell: bash
+        run: |
+          if ! qlt bundle init --use-runner ubuntu-latest --language cpp --automation-type actions --development --overwrite-existing ; then
+            echo "Failed to generate bundle workflow files."
+            exit 1
+          fi
+
+
       - name: Check Git Clean Status 
         shell: bash
         run: |

.github/workflows/run-bundle-integration-tests-cpp.yml

@@ -0,0 +1,123 @@
+name: ⚙️ Integration Test Bundle (cpp)
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  integration-test:
+    name: Run Bundle Integration Test
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+
+    - name: Install QLT
+      id: install-qlt
+      uses: ./.github/actions/install-qlt-local
+      with:
+        qlt-version: 'latest'
+        add-to-path: true
+
+
+    - name: Validate QLT Installation
+      shell: bash
+      run: |
+        echo -e "Checking QLT Version:"
+        echo "QLT Home: ${{ steps.install-qlt.outputs.qlt-home }}"
+        qlt version
+
+
+    - name: Create Bundle (compiled)
+      shell: bash
+      run: |
+        if ! qlt codeql run install --custom-bundle --base example/  ; then
+          echo "Failed to generate bundle."
+          exit 1
+        fi
+
+        # ensure bundle runs 
+
+        if ! qlt query run install-packs --use-bundle --base example/  ; then
+          echo "Failed to install query packs with tool."
+          exit 1
+        fi
+
+
+    - name: Validate Bundle Existence
+      shell: bash
+      run: |
+        echo "Checking Bundle Existence"
+        ls -l ${{ env.QLT_CODEQL_HOME }}/../out/
+
+    - name: Upload Bundle Used
+      uses: actions/upload-artifact@v2
+      with:
+        name: codeql-bundle.tar.gz
+        path: |
+          ${{ env.QLT_CODEQL_BUNDLE_PATH }}
+        if-no-files-found: error
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        tools: ${{ env.QLT_CODEQL_BUNDLE_PATH }}
+
+
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+      with:
+        working-directory: example/integration-tests/${{ matrix.language }}/src/  # Path containing the example application 
+
+
+
+    - name: Perform CodeQL Analysis
+      id: analysis
+      uses: github/codeql-action/analyze@v2
+
+    - name: Validate SARIF Location
+      shell: bash
+      run: |
+        # validate we have the actual sarif results
+        echo "Checking SARIF file location at: ${{ steps.analysis.outputs.sarif-output }}"
+        ls -l ${{ steps.analysis.outputs.sarif-output }}
+
+    - name: Upload SARIF Results
+      uses: actions/upload-artifact@v2
+      with:
+        name: actual.sarif
+        path: |
+          ${{ steps.analysis.outputs.sarif-output }}/*.sarif
+        if-no-files-found: error
+
+    - name: Validate SARIF Existence
+      shell: bash
+      run: |        
+        ls -l ${{ steps.analysis.outputs.sarif-output }}/*.sarif
+
+
+
+
+    - name: Validate SARIF Results
+      shell: bash
+      run: |
+        # Compare the expected vs the actual 
+        qlt bundle run validate-integration-tests --expected example/integration-tests/${{ matrix.language }}/expected.sarif --actual ${{ steps.analysis.outputs.sarif-output }}/${{ matrix.language }}.sarif        
+

example/qlt.conf.json

@@ -4,14 +4,15 @@
   "CodeQLCLIBundle": "codeql-bundle-v2.15.5",
   "EnableCustomCodeQLBundles": true,
   "CodeQLStandardLibraryIdent": "codeql-cli_v2.15.5",
-  "CustomizationPacks" : [
+  "CodeQLPackConfiguration" : [
     {
       "Name": "qlt/cpp-customizations",
-      "Export" : true
+      "Bundle" : true
     },
     {
       "Name": "qlt2/stuff2-tests",
-      "Export" : false
+      "Bundle" : false,
+      "ReferencesBundle" : true 
     }
   ]
 }

src/CodeQLToolkit.Core/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "CodeQLToolkit.Core": {
       "commandName": "Project",
-      "commandLineArgs": "--base C:\\Projects\\codeql-development-lifecycle-toolkit\\example  bundle set enable-custom-bundles"
+      "commandLineArgs": "bundle init --use-runner ubuntu-latest --language cpp --automation-type actions --development --overwrite-existing"
     }
   }
 }

src/CodeQLToolkit.Core/ver.txt

@@ -1 +1 @@
-0.0.17
+0.0.23

src/CodeQLToolkit.Features/Bundle/Lifecycle/BaseLifecycleTarget.cs

@@ -8,12 +8,7 @@ namespace CodeQLToolkit.Features.Bundle.Lifecycle
 {
     abstract public class BaseLifecycleTarget : ILifecycleTarget
     {
-        public int NumThreads { get; set; }     
         public string UseRunner { get; set; }   
 
-        public string ExtraArgs { get; set; }
-
-
-
     }
 }

src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs

@@ -1,10 +1,6 @@
-using CodeQLToolkit.Features.CodeQL.Lifecycle.Targets;
-using CodeQLToolkit.Features.CodeQL.Lifecycle;
-using CodeQLToolkit.Features.Test.Lifecycle.Targets;
-using CodeQLToolkit.Features.Test.Lifecycle.Targets.Actions;
+using CodeQLToolkit.Features.CodeQL.Lifecycle;
 using CodeQLToolkit.Shared.Utils;
 using System.CommandLine;
-using System.Reflection;
 using CodeQLToolkit.Features.Bundle.Lifecycle.Targets;
 
 namespace CodeQLToolkit.Features.Bundle.Lifecycle
@@ -32,7 +28,18 @@ public override LanguageType[] SupportedLangauges
 
         public void Register(Command parentCommand)
         {
-            //Log<BundleLifecycleFeature>.G().LogInformation("Registering lifecycle submodule.");
+            Log<BundleLifecycleFeature>.G().LogInformation("Registering lifecycle submodule.");
+
+            var initCommand = new Command("init", "Initialize bundle creation and integration testing features.");
+            var overwriteExistingOption = new Option<bool>("--overwrite-existing", () => false, "Overwrite exiting files (if they exist).");
+            var useRunnerOption = new Option<string>("--use-runner", () => "ubuntu-latest", "The runner(s) to use. Should be a comma-seperated list of actions runners.");
+            var languageOption = new Option<string>("--language", $"The language to generate automation for.") { IsRequired = true }.FromAmong(SupportedLangauges.Select(x => x.ToOptionString()).ToArray());
+
+            initCommand.AddOption(overwriteExistingOption);
+            initCommand.AddOption(useRunnerOption);
+            initCommand.AddOption(languageOption);
+
+            parentCommand.Add(initCommand);
 
             var setCommand = new Command("set", "Functions pertaining to setting variables related to custom CodeQL bundles.");
             //parentCommand.Add(setCommand);
@@ -90,6 +97,27 @@ public void Register(Command parentCommand)
             }
 
 
+            initCommand.SetHandler((devMode, basePath, automationType, overwriteExisting, useRunner, language) =>
+            {
+                Log<BundleLifecycleFeature>.G().LogInformation("Executing init command...");
+
+                //
+                // dispatch at runtime to the correct automation type
+                //
+                var featureTarget = AutomationFeatureFinder.FindTargetForAutomationType<BaseLifecycleTarget>(AutomationTypeHelper.AutomationTypeFromString(automationType));
+
+                // setup common params 
+                featureTarget.FeatureName = FeatureName;
+                featureTarget.Base = basePath;
+                featureTarget.OverwriteExisting = overwriteExisting;
+                featureTarget.UseRunner = useRunner;
+                featureTarget.Language = language;
+                featureTarget.DevMode = devMode;
+                featureTarget.Run();
+
+            }, Globals.Development, Globals.BasePathOption, Globals.AutomationTypeOption, overwriteExistingOption, useRunnerOption, languageOption);
+
+
         }
 
         public int Run()

src/CodeQLToolkit.Features/Bundle/Lifecycle/Targets/Actions/InitLifecycleTarget.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodeQLToolkit.Features.Bundle.Lifecycle.Targets.Actions
+{
+    [AutomationType(AutomationType.ACTIONS)]
+    public class InitLifecycleTarget : BaseLifecycleTarget
+    {
+
+        public InitLifecycleTarget()
+        {
+            AutomationType = AutomationType.ACTIONS;
+        }
+
+        public override void Run()
+        {
+            Log<InitLifecycleTarget>.G().LogInformation("Running init command...");
+
+            // temporarily disable the language resolution 
+            var tmpLanguage = Language;
+            Language = null;
+
+            WriteTemplateIfOverwriteOrNotExists("install-qlt", Path.Combine(Base, ".github", "actions", "install-qlt", "action.yml"), "install-qlt action");
+            WriteTemplateIfOverwriteOrNotExists("run-bundle-integration-tests", Path.Combine(Base, ".github", "workflows", $"run-bundle-integration-tests-{tmpLanguage}.yml"), $"Run CodeQL Unit Tests ({Language})", new
+            {
+                useRunner = UseRunner,
+                language = tmpLanguage,
+                devMode = DevMode,
+            });
+
+            Language = tmpLanguage;
+
+            var message = @"------------------------------------------
+Your repository now has the Bundle Creation and Integration Test Runner installed in `.github/workflows/`. Additionally, 
+QLT has installed necessary actions for keeping your version of QLT and CodeQL current in `.github/actions/install-qlt`. 
+
+Note that for integration testing to work, you MUST create a directory `integration-test` in the root of your repository. Please
+consult the QLT documentation for details on how to structure this directory. 
+
+In addition to using QLT to generate your files you can also directly edit this file to fine tune its settings. 
+            
+(Hint: If you'd like to regenerate your files, you can use the `--overwrite-existing` option to overwrite the files that are in place now.)";
+
+            Log<InitLifecycleTarget>.G().LogInformation(message);
+        }
+    }
+}

src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs

@@ -27,9 +27,10 @@ public override void Run()
                 if (Packs!=null && Packs.Length > 0)
                 {
                     Log<InstallCommand>.G().LogInformation($"Overriding Packs on the command line. The following Packs will be packaged:");
-                    installation.CustomizationPacks = Packs.Select(p => new QLTCustomizationPack()
+                    installation.CodeQLPackConfiguration = Packs.Select(p => new CodeQLPackConfiguration()
                     {
-                        Name = p
+                        Name = p,
+                        Bundle = true
                     }).ToArray();
                 }
                 else

src/CodeQLToolkit.Features/CodeQLToolkit.Features.csproj

@@ -11,7 +11,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Folder Include="Bundle\Lifecycle\Targets\Actions\" />
     <Folder Include="Bundle\Models\" />
     <Folder Include="Pack\" />
     <Folder Include="Templates\Validation\Actions\" />
@@ -22,6 +21,12 @@
   </ItemGroup>
 
   <ItemGroup>
+    <None Update="Templates\Bundle\Actions\install-qlt.liquid">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Templates\Bundle\Actions\run-bundle-integration-tests.liquid">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Update="Templates\Query\codeql-workspace.liquid">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>

src/CodeQLToolkit.Features/Query/Commands/Targets/InstallQueryPacksCommandTarget.cs

@@ -41,15 +41,15 @@ public override void Run()
                 Log<InstallQueryPacksCommandTarget>.G().LogInformation("In bundle mode so filtering bundled packs...");
 
 
-                foreach (var pack in config.CustomizationPacks)
+                foreach (var pack in config.CodeQLPackConfiguration)
                 {
                     Log<InstallQueryPacksCommandTarget>.G().LogInformation($"Pack {pack.Name} will NOT installed because it is part of the bundle...");
                 }
 
                 files = files.Where(f => 
                     // all things that are part of the customization pack must be excluded. 
                     // if it is exported is not relevant here.
-                    !config.CustomizationPacks.Any(p => CodeQLPackReader.read(f).Name == p.Name)
+                    !config.CodeQLPackConfiguration.Any(p => CodeQLPackReader.read(f).Name == p.Name && (p.Bundle==true || p.ReferencesBundle==true))
                 ).ToArray();
 
                 Log<InstallQueryPacksCommandTarget>.G().LogInformation($"Got {files.Length} packs after filtering...");