Support multi-platform bundles and refactor

Custom bundle creation now uses a platform-independent CodeQL base bundle and creates separate output bundles for Windows, Linux, and MacOS. These bundles are exposed via the QLT_CODEQL_BUNDLE_PATH_WIN64, QLT_CODEQL_BUNDLE_PATH_LINUX64, and QLT_CODEQL_BUNDLE_PATH_OSX64 environment variables, with QLT_CODEQL_BUNDLE_PATH continuing to map to the bundle compatible with the runner platform.

.github/actions/install-qlt-local/action.yml

@@ -60,7 +60,7 @@ runs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
+        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
       env:
         GH_TOKEN: ${{ github.token }}
 

.github/workflows/internal-build-release-linux64.yml

@@ -50,7 +50,7 @@ jobs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
+        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/
       env:
         GH_TOKEN: ${{ github.token }}
 

.github/workflows/internal-build-release-macos64.yml

@@ -48,7 +48,7 @@ jobs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/
+        ./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/
       env:
         GH_TOKEN: ${{ github.token }}
 

.github/workflows/internal-build-release-win64.yml

@@ -45,7 +45,7 @@ jobs:
         pip install -U pyinstaller 
 
         # run the packaging 
-        .\scripts\build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\
+        .\scripts\build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\
       env:
         GH_TOKEN: ${{ github.token }}
 

.github/workflows/internal-pr-bundle-integration-test-cpp.yml

@@ -92,13 +92,16 @@ jobs:
           ${{ steps.analysis.outputs.sarif-output }}/*.sarif
         if-no-files-found: error
 
-    - name: Upload Bundle Used
+    - name: Upload Bundles
       uses: actions/upload-artifact@v2
       with:
-        name: codeql-bundle.tar.gz
+        name: codeql-bundles
         path: |
-          ${{ env.QLT_CODEQL_BUNDLE_PATH }}
+          ${{ env.QLT_CODEQL_BUNDLE_PATH_LINUX64 }}
+          ${{ env.QLT_CODEQL_BUNDLE_PATH_WIN64 }}
+          ${{ env.QLT_CODEQL_BUNDLE_PATH_OSX64 }}
         if-no-files-found: error
+        compression-level: 0
 
     - name: Validate SARIF Results
       shell: bash

.github/workflows/run-bundle-integration-tests-cpp.yml

@@ -65,13 +65,16 @@ jobs:
         echo "Checking Bundle Existence"
         ls -l ${{ env.QLT_CODEQL_HOME }}/../out/
 
-    - name: Upload Bundle Used
+    - name: Upload Bundles
       uses: actions/upload-artifact@v2
       with:
-        name: codeql-bundle.tar.gz
+        name: codeql-bundles
         path: |
-          ${{ env.QLT_CODEQL_BUNDLE_PATH }}
+          ${{ env.QLT_CODEQL_BUNDLE_PATH_LINUX64 }}
+          ${{ env.QLT_CODEQL_BUNDLE_PATH_WIN64 }}
+          ${{ env.QLT_CODEQL_BUNDLE_PATH_OSX64 }}
         if-no-files-found: error
+        compression-level: 0
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

developer_guide.md

@@ -15,7 +15,7 @@ Note that we keep recent copies of tools (for local debugging purposes) in the `
 **CodeQL Bundle**
 
 ```
-./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools
+./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools
 ```
 
 

scripts/build_codeql_bundle_dist.ps1

@@ -1,12 +1,12 @@
 param(
-    [Parameter(Mandatory=$true)] 
+    [Parameter(Mandatory = $true)] 
     [string]
     $Version,
-    [Parameter(Mandatory=$true)] 
+    [Parameter(Mandatory = $true)] 
     [string]
     $WorkDirectory,      
 
-    [Parameter(Mandatory=$true)] 
+    [Parameter(Mandatory = $true)] 
     [string]
     $DestinationDirectory       
 )
@@ -20,7 +20,7 @@ if (-not (Test-Path $DestinationDirectory)) {
 }
 
 # download a copy of the release from GitHub
-gh release download "v$Version" --repo https://github.com/jsinglet/codeql-bundle  -D $WorkDirectory -A zip
+gh release download "v$Version" --repo https://github.com/kraiouchkine/codeql-bundle  -D $WorkDirectory -A zip
 
 # extract the zip file
 Expand-Archive -Path "$WorkDirectory\codeql-bundle-$Version.zip" -DestinationPath $WorkDirectory
@@ -45,9 +45,10 @@ pyinstaller -F -n codeql_bundle cli.py
 Pop-Location 
 Pop-Location 
 
-if($IsWindows){
+if ($IsWindows) {
     $OutputFile = Join-Path $ArchiveDirectory "codeql_bundle" "dist" "codeql_bundle.exe"
-}else{
+}
+else {
     $OutputFile = Join-Path $ArchiveDirectory "codeql_bundle" "dist" "codeql_bundle"
 }
 

src/CodeQLToolkit.Features/Bundle/BundleFeatureMain.cs

@@ -29,7 +29,7 @@ private BundleFeatureMain()
             commandFeature = new BundleCommandFeature();
             lifecycleFeature = new BundleLifecycleFeature();
         }
-        public static BundleFeatureMain Instance { get { return instance; } }
+        public static BundleFeatureMain Instance => instance;
 
         public void Register(Command parentCommand)
         {

src/CodeQLToolkit.Features/Bundle/Commands/BundleCommandFeature.cs

@@ -11,9 +11,7 @@ namespace CodeQLToolkit.Features.Bundle.Commands
 {
     public class BundleCommandFeature : FeatureBase, IToolkitLifecycleFeature
     {
-        public override LanguageType[] SupportedLangauges
-        {
-            get => new LanguageType[] {
+        public override LanguageType[] SupportedLangauges => new LanguageType[] {
             LanguageType.C,
             LanguageType.CPP,
             LanguageType.CSHARP,
@@ -23,7 +21,6 @@ public override LanguageType[] SupportedLangauges
             LanguageType.RUBY,
             LanguageType.PYTHON
         };
-        }
 
         public BundleCommandFeature()
         {
@@ -62,7 +59,7 @@ public void Register(Command parentCommand)
 
                 }.Run();
 
-            },Globals.BasePathOption, expectedOption, actualOption);
+            }, Globals.BasePathOption, expectedOption, actualOption);
         }
 
         public int Run()

src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs

@@ -7,14 +7,12 @@ namespace CodeQLToolkit.Features.Bundle.Lifecycle
 {
     public class BundleLifecycleFeature : FeatureBase, IToolkitLifecycleFeature
     {
-        public BundleLifecycleFeature() 
+        public BundleLifecycleFeature()
         {
             FeatureName = "Bundle";
         }
 
-        public override LanguageType[] SupportedLangauges
-        {
-            get => new LanguageType[] {
+        public override LanguageType[] SupportedLangauges => new LanguageType[] {
             LanguageType.C,
             LanguageType.CPP,
             LanguageType.CSHARP,
@@ -24,7 +22,6 @@ public override LanguageType[] SupportedLangauges
             LanguageType.RUBY,
             LanguageType.PYTHON
         };
-        }
 
         public void Register(Command parentCommand)
         {
@@ -44,7 +41,7 @@ public void Register(Command parentCommand)
             var setCommand = new Command("set", "Functions pertaining to setting variables related to custom CodeQL bundles.");
             //parentCommand.Add(setCommand);
 
-            var enableCommand = new Command("enable-custom-bundles", "Enables custom CodeQL Bundles.");            
+            var enableCommand = new Command("enable-custom-bundles", "Enables custom CodeQL Bundles.");
             //setCommand.Add(enableCommand);
 
             var disableCommand = new Command("disable-custom-bundles", "Disables custom CodeQL Bundles.");

src/CodeQLToolkit.Features/CodeQL/CodeQLFeatureMain.cs

@@ -7,7 +7,7 @@
 
 namespace CodeQLToolkit.Features.CodeQL
 {
-        public class CodeQLFeatureMain : IToolkitFeature
+    public class CodeQLFeatureMain : IToolkitFeature
     {
         readonly CodeQLLifecycleFeature lifecycleFeature;
         readonly CodeQLCommandFeature commandFeature;
@@ -24,7 +24,7 @@ private CodeQLFeatureMain()
             commandFeature = new CodeQLCommandFeature();
         }
 
-        public static CodeQLFeatureMain Instance { get { return instance; } }
+        public static CodeQLFeatureMain Instance => instance;
 
         public int Run()
         {
@@ -36,7 +36,7 @@ public void Register(Command parentCommand)
         {
             var queryCommand = new Command("codeql", "Use the features related to managing the version of CodeQL used by this repository.");
             parentCommand.Add(queryCommand);
-            
+
             Log<CodeQLFeatureMain>.G().LogInformation("Registering scaffolding submodule.");
             lifecycleFeature.Register(queryCommand);
 

src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs

@@ -15,16 +15,16 @@ namespace CodeQLToolkit.Features.CodeQL.Commands
 {
     public class CodeQLCommandFeature : FeatureBase, IToolkitLifecycleFeature
     {
-        public override LanguageType[] SupportedLangauges { get => new LanguageType[] { 
+        public override LanguageType[] SupportedLangauges => new LanguageType[] {
             LanguageType.C,
             LanguageType.CPP,
             LanguageType.CSHARP,
             LanguageType.JAVA,
             LanguageType.JAVASCRIPT,
             LanguageType.GO,
             LanguageType.RUBY,
-            LanguageType.PYTHON            
-        }; }
+            LanguageType.PYTHON
+        };
 
         public CodeQLCommandFeature()
         {
@@ -37,18 +37,18 @@ public void Register(Command parentCommand)
 
             var runCommand = new Command("run", "Functions pertaining to running codeql-related commands.");
             parentCommand.Add(runCommand);
-            
+
             var installCommand = new Command("install", "Installs CodeQL (bundle or release distribution) locally.");
-            var customBundleOption = new Option<bool>("--custom-bundle", () => false, "Build a custom bundle and compile the bundle.") { IsRequired = true};
-            var quickBundleOption = new Option<bool>("--quick-bundle", () => false,  "Build a custom bundle and DO NOT compile the bundle.") { IsRequired = true};
+            var customBundleOption = new Option<bool>("--custom-bundle", () => false, "Build a custom bundle and compile the bundle.") { IsRequired = true };
+            var quickBundleOption = new Option<bool>("--quick-bundle", () => false, "Build a custom bundle and DO NOT compile the bundle.") { IsRequired = true };
             var packsOption = new Option<string[]>("--packs", "When creating bundles, this specifies the packs to include, Example `pack1 pack2 pack3`. You may specify also as `--pack pack1 --pack2 --pack3`") { IsRequired = false, AllowMultipleArgumentsPerToken = true };
 
             installCommand.Add(customBundleOption);
             installCommand.Add(quickBundleOption);
             installCommand.Add(packsOption);
 
             runCommand.Add(installCommand);
-            
+
             installCommand.SetHandler((basePath, automationType, customBundleOption, quickBundleOption, packs) =>
             {
                 Log<CodeQLCommandFeature>.G().LogInformation("Executing install command...");

src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs

@@ -15,16 +15,45 @@ public class InstallCommand : CommandTarget
         public bool CustomBundles { get; set; }
         public bool QuickBundles { get; set; }
         public string[] Packs { get; set; }
+
+        void SetEnvironmentVariableMultiTarget(string name, string value)
+        {
+            Log<InstallCommand>.G().LogInformation($"Setting {name} to {value}...");
+
+            Environment.SetEnvironmentVariable(name, value);
+
+            if (AutomationTypeHelper.AutomationTypeFromString(AutomationTarget) == AutomationType.ACTIONS)
+            {
+                string? githubEnvPath = Environment.GetEnvironmentVariable("GITHUB_ENV");
+                try
+                {
+                    if (File.Exists(githubEnvPath))
+                    {
+                        File.AppendAllText(githubEnvPath, $"{name}={value}\n");
+                    }
+                    else
+                    {
+                        throw new Exception("Could not find GITHUB_ENV file.");
+                    }
+                }
+                catch (Exception)
+                {
+                    Log<InstallCommand>.G().LogError($"Could not write to GITHUB_ENV file.");
+                    throw;
+                }
+            }
+        }
+
         public override void Run()
         {
             Log<InstallCommand>.G().LogInformation($"Running Install command");
 
             // First, check if CodeQL is installed.
             var installation = CodeQLInstallation.LoadFromConfig(Base);
-            if(CustomBundles || QuickBundles)
+            if (CustomBundles || QuickBundles)
             {
                 installation.EnableCustomCodeQLBundles = true;
-                if (Packs!=null && Packs.Length > 0)
+                if (Packs != null && Packs.Length > 0)
                 {
                     Log<InstallCommand>.G().LogInformation($"Overriding Packs on the command line. The following Packs will be packaged:");
                     installation.CodeQLPackConfiguration = Packs.Select(p => new CodeQLPackConfiguration()
@@ -40,14 +69,12 @@ public override void Run()
 
                 installation.LogPacksToBeBuilt();
 
-
                 installation.QuickBundle = QuickBundles;
             }
 
             Log<InstallCommand>.G().LogInformation($"Checking for installation...");
 
-            // if it is the case that it is installed but we are in custom bundle mode we RE install it. 
-
+            // If CodeQL is already installed, but custom bundles are enabled, reinstall CodeQL anyway to ensure use of the correct custom bundle.
             if (installation.IsInstalled() && !installation.EnableCustomCodeQLBundles)
             {
                 Log<InstallCommand>.G().LogInformation($"CodeQL is already installed at that version. Please delete the installation directory to reinstall.");
@@ -57,39 +84,19 @@ public override void Run()
                 Log<InstallCommand>.G().LogInformation($"Installing CodeQL...");
                 installation.Install();
 
-                // set the environment variable
-                Log<InstallCommand>.G().LogInformation($"Setting QLT_CODEQL_HOME to {installation.CodeQLHome}...");
-                Log<InstallCommand>.G().LogInformation($"Setting QLT_CODEQL_PATH to {installation.CodeQLToolBinary}...");
+                SetEnvironmentVariableMultiTarget("QLT_CODEQL_HOME", installation.CodeQLHome);
+                SetEnvironmentVariableMultiTarget("QLT_CODEQL_PATH", installation.CodeQLToolBinary);
 
-                Environment.SetEnvironmentVariable("QLT_CODEQL_HOME", installation.CodeQLHome);
-                Environment.SetEnvironmentVariable("QLT_CODEQL_PATH", installation.CodeQLToolBinary);
                 if (CustomBundles || QuickBundles)
                 {
-                    Environment.SetEnvironmentVariable("QLT_CODEQL_BUNDLE_PATH", installation.CustomBundleOutputBundle);
-                }
-
-                if (AutomationTypeHelper.AutomationTypeFromString(AutomationTarget) == AutomationType.ACTIONS)
-                {
-                    if (Environment.GetEnvironmentVariable("GITHUB_ENV") != null && File.Exists(Environment.GetEnvironmentVariable("GITHUB_ENV")))
-                    {
-
-                        File.AppendAllText(Environment.GetEnvironmentVariable("GITHUB_ENV"), $"QLT_CODEQL_HOME={installation.CodeQLHome}" + "\n");
-                        File.AppendAllText(Environment.GetEnvironmentVariable("GITHUB_ENV"), $"QLT_CODEQL_PATH={installation.CodeQLToolBinary}" + "\n");
-                        if (CustomBundles || QuickBundles)
-                        {
-                            File.AppendAllText(Environment.GetEnvironmentVariable("GITHUB_ENV"), $"QLT_CODEQL_BUNDLE_PATH={installation.CustomBundleOutputBundle}" + "\n");
-                        }
-                    }
+                    SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH", installation.CustomBundleOutputBundleCurrentPlatform);
+                    SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH_WIN64", installation.CustomBundleOutputBundleWindows);
+                    SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH_OSX64", installation.CustomBundleOutputBundleOSX);
+                    SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH_LINUX64", installation.CustomBundleOutputBundleLinux);
                 }
-
             }
 
-
             Log<InstallCommand>.G().LogInformation($"Done.");
-
-
-
-
         }
     }
 }

src/CodeQLToolkit.Features/CodeQL/Lifecycle/Targets/GetVersionLifecycleTarget.cs

@@ -9,7 +9,7 @@ namespace CodeQLToolkit.Features.CodeQL.Lifecycle.Targets
 {
     public class GetVersionLifecycleTarget : ILifecycleTarget
     {
-       
+
         override public void Run()
         {
             Log<GetVersionLifecycleTarget>.G().LogInformation("Running get command...");
@@ -19,11 +19,11 @@ override public void Run()
                 Base = Base
             };
 
-            if (!File.Exists(c.CodeQLConfigFilePath))
+            if (!File.Exists(c.QLTConfigFilePath))
             {
-                ProcessUtils.DieWithError($"Cannot read values from missing file {c.CodeQLConfigFilePath}");
+                ProcessUtils.DieWithError($"Cannot read values from missing file {c.QLTConfigFilePath}");
             }
-           
+
             var config = c.FromFile();
 
             // This should be updated so that we can pretty print all the various options: