diff --git a/.github/actions/install-qlt-local/action.yml b/.github/actions/install-qlt-local/action.yml index 2c0d61a..1fbcfde 100644 --- a/.github/actions/install-qlt-local/action.yml +++ b/.github/actions/install-qlt-local/action.yml @@ -60,7 +60,7 @@ runs: pip install -U pyinstaller # run the packaging - ./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ + ./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/internal-build-release-linux64.yml b/.github/workflows/internal-build-release-linux64.yml index 51373b2..5a9dc7b 100644 --- a/.github/workflows/internal-build-release-linux64.yml +++ b/.github/workflows/internal-build-release-linux64.yml @@ -50,7 +50,7 @@ jobs: pip install -U pyinstaller # run the packaging - ./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ + ./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/internal-build-release-macos64.yml b/.github/workflows/internal-build-release-macos64.yml index 17cea86..5f63213 100644 --- a/.github/workflows/internal-build-release-macos64.yml +++ b/.github/workflows/internal-build-release-macos64.yml @@ -48,7 +48,7 @@ jobs: pip install -U pyinstaller # run the packaging - ./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/ + ./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/ env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/internal-build-release-win64.yml b/.github/workflows/internal-build-release-win64.yml index ddeda75..58f73b6 100644 --- a/.github/workflows/internal-build-release-win64.yml +++ b/.github/workflows/internal-build-release-win64.yml @@ -45,7 +45,7 @@ jobs: pip install -U pyinstaller # run the packaging - .\scripts\build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\ + .\scripts\build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\ env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/internal-pr-bundle-integration-test-cpp.yml b/.github/workflows/internal-pr-bundle-integration-test-cpp.yml index db691f1..d78b7ee 100644 --- a/.github/workflows/internal-pr-bundle-integration-test-cpp.yml +++ b/.github/workflows/internal-pr-bundle-integration-test-cpp.yml @@ -92,13 +92,16 @@ jobs: ${{ steps.analysis.outputs.sarif-output }}/*.sarif if-no-files-found: error - - name: Upload Bundle Used + - name: Upload Bundles uses: actions/upload-artifact@v2 with: - name: codeql-bundle.tar.gz + name: codeql-bundles path: | - ${{ env.QLT_CODEQL_BUNDLE_PATH }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_LINUX64 }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_WIN64 }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_OSX64 }} if-no-files-found: error + compression-level: 0 - name: Validate SARIF Results shell: bash diff --git a/.github/workflows/run-bundle-integration-tests-cpp.yml b/.github/workflows/run-bundle-integration-tests-cpp.yml index f1c66ab..f3569c6 100644 --- a/.github/workflows/run-bundle-integration-tests-cpp.yml +++ b/.github/workflows/run-bundle-integration-tests-cpp.yml @@ -52,7 +52,6 @@ jobs: fi # ensure bundle runs - if ! qlt query run install-packs --use-bundle --base example/ ; then echo "Failed to install query packs with tool." exit 1 @@ -65,13 +64,16 @@ jobs: echo "Checking Bundle Existence" ls -l ${{ env.QLT_CODEQL_HOME }}/../out/ - - name: Upload Bundle Used + - name: Upload Bundles uses: actions/upload-artifact@v2 with: - name: codeql-bundle.tar.gz + name: codeql-bundles path: | - ${{ env.QLT_CODEQL_BUNDLE_PATH }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_LINUX64 }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_WIN64 }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_OSX64 }} if-no-files-found: error + compression-level: 0 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/developer_guide.md b/developer_guide.md index 18c690a..674d4bd 100644 --- a/developer_guide.md +++ b/developer_guide.md @@ -15,7 +15,7 @@ Note that we keep recent copies of tools (for local debugging purposes) in the ` **CodeQL Bundle** ``` -./scripts/build_codeql_bundle_dist.ps1 -Version 0.2.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools +./scripts/build_codeql_bundle_dist.ps1 -Version 0.3.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools ``` diff --git a/scripts/build_codeql_bundle_dist.ps1 b/scripts/build_codeql_bundle_dist.ps1 index 4a71c5a..1e9547e 100644 --- a/scripts/build_codeql_bundle_dist.ps1 +++ b/scripts/build_codeql_bundle_dist.ps1 @@ -1,12 +1,12 @@ param( - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string] $Version, - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string] $WorkDirectory, - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string] $DestinationDirectory ) @@ -20,7 +20,7 @@ if (-not (Test-Path $DestinationDirectory)) { } # download a copy of the release from GitHub -gh release download "v$Version" --repo https://github.com/jsinglet/codeql-bundle -D $WorkDirectory -A zip +gh release download "v$Version" --repo https://github.com/kraiouchkine/codeql-bundle -D $WorkDirectory -A zip # extract the zip file Expand-Archive -Path "$WorkDirectory\codeql-bundle-$Version.zip" -DestinationPath $WorkDirectory @@ -45,9 +45,10 @@ pyinstaller -F -n codeql_bundle cli.py Pop-Location Pop-Location -if($IsWindows){ +if ($IsWindows) { $OutputFile = Join-Path $ArchiveDirectory "codeql_bundle" "dist" "codeql_bundle.exe" -}else{ +} +else { $OutputFile = Join-Path $ArchiveDirectory "codeql_bundle" "dist" "codeql_bundle" } diff --git a/src/CodeQLToolkit.Features/Bundle/BundleFeatureMain.cs b/src/CodeQLToolkit.Features/Bundle/BundleFeatureMain.cs index d55f223..c5ae7b6 100644 --- a/src/CodeQLToolkit.Features/Bundle/BundleFeatureMain.cs +++ b/src/CodeQLToolkit.Features/Bundle/BundleFeatureMain.cs @@ -29,7 +29,7 @@ private BundleFeatureMain() commandFeature = new BundleCommandFeature(); lifecycleFeature = new BundleLifecycleFeature(); } - public static BundleFeatureMain Instance { get { return instance; } } + public static BundleFeatureMain Instance => instance; public void Register(Command parentCommand) { diff --git a/src/CodeQLToolkit.Features/Bundle/Commands/BundleCommandFeature.cs b/src/CodeQLToolkit.Features/Bundle/Commands/BundleCommandFeature.cs index f6fb7fc..efc9bdb 100644 --- a/src/CodeQLToolkit.Features/Bundle/Commands/BundleCommandFeature.cs +++ b/src/CodeQLToolkit.Features/Bundle/Commands/BundleCommandFeature.cs @@ -11,9 +11,7 @@ namespace CodeQLToolkit.Features.Bundle.Commands { public class BundleCommandFeature : FeatureBase, IToolkitLifecycleFeature { - public override LanguageType[] SupportedLangauges - { - get => new LanguageType[] { + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.CSHARP, @@ -23,7 +21,6 @@ public override LanguageType[] SupportedLangauges LanguageType.RUBY, LanguageType.PYTHON }; - } public BundleCommandFeature() { @@ -62,7 +59,7 @@ public void Register(Command parentCommand) }.Run(); - },Globals.BasePathOption, expectedOption, actualOption); + }, Globals.BasePathOption, expectedOption, actualOption); } public int Run() diff --git a/src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs b/src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs index e8234cb..e53431a 100644 --- a/src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs +++ b/src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs @@ -7,14 +7,12 @@ namespace CodeQLToolkit.Features.Bundle.Lifecycle { public class BundleLifecycleFeature : FeatureBase, IToolkitLifecycleFeature { - public BundleLifecycleFeature() + public BundleLifecycleFeature() { FeatureName = "Bundle"; } - public override LanguageType[] SupportedLangauges - { - get => new LanguageType[] { + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.CSHARP, @@ -24,7 +22,6 @@ public override LanguageType[] SupportedLangauges LanguageType.RUBY, LanguageType.PYTHON }; - } public void Register(Command parentCommand) { @@ -44,7 +41,7 @@ public void Register(Command parentCommand) var setCommand = new Command("set", "Functions pertaining to setting variables related to custom CodeQL bundles."); //parentCommand.Add(setCommand); - var enableCommand = new Command("enable-custom-bundles", "Enables custom CodeQL Bundles."); + var enableCommand = new Command("enable-custom-bundles", "Enables custom CodeQL Bundles."); //setCommand.Add(enableCommand); var disableCommand = new Command("disable-custom-bundles", "Disables custom CodeQL Bundles."); diff --git a/src/CodeQLToolkit.Features/CodeQL/CodeQLFeatureMain.cs b/src/CodeQLToolkit.Features/CodeQL/CodeQLFeatureMain.cs index ee51b92..b68a797 100644 --- a/src/CodeQLToolkit.Features/CodeQL/CodeQLFeatureMain.cs +++ b/src/CodeQLToolkit.Features/CodeQL/CodeQLFeatureMain.cs @@ -7,7 +7,7 @@ namespace CodeQLToolkit.Features.CodeQL { - public class CodeQLFeatureMain : IToolkitFeature + public class CodeQLFeatureMain : IToolkitFeature { readonly CodeQLLifecycleFeature lifecycleFeature; readonly CodeQLCommandFeature commandFeature; @@ -24,7 +24,7 @@ private CodeQLFeatureMain() commandFeature = new CodeQLCommandFeature(); } - public static CodeQLFeatureMain Instance { get { return instance; } } + public static CodeQLFeatureMain Instance => instance; public int Run() { @@ -36,7 +36,7 @@ public void Register(Command parentCommand) { var queryCommand = new Command("codeql", "Use the features related to managing the version of CodeQL used by this repository."); parentCommand.Add(queryCommand); - + Log.G().LogInformation("Registering scaffolding submodule."); lifecycleFeature.Register(queryCommand); diff --git a/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs b/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs index 1138781..7ed70f9 100644 --- a/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs +++ b/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs @@ -15,7 +15,7 @@ namespace CodeQLToolkit.Features.CodeQL.Commands { public class CodeQLCommandFeature : FeatureBase, IToolkitLifecycleFeature { - public override LanguageType[] SupportedLangauges { get => new LanguageType[] { + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.CSHARP, @@ -23,8 +23,8 @@ public class CodeQLCommandFeature : FeatureBase, IToolkitLifecycleFeature LanguageType.JAVASCRIPT, LanguageType.GO, LanguageType.RUBY, - LanguageType.PYTHON - }; } + LanguageType.PYTHON + }; public CodeQLCommandFeature() { @@ -37,10 +37,10 @@ public void Register(Command parentCommand) var runCommand = new Command("run", "Functions pertaining to running codeql-related commands."); parentCommand.Add(runCommand); - + var installCommand = new Command("install", "Installs CodeQL (bundle or release distribution) locally."); - var customBundleOption = new Option("--custom-bundle", () => false, "Build a custom bundle and compile the bundle.") { IsRequired = true}; - var quickBundleOption = new Option("--quick-bundle", () => false, "Build a custom bundle and DO NOT compile the bundle.") { IsRequired = true}; + var customBundleOption = new Option("--custom-bundle", () => false, "Build a custom bundle and compile the bundle.") { IsRequired = true }; + var quickBundleOption = new Option("--quick-bundle", () => false, "Build a custom bundle and DO NOT compile the bundle.") { IsRequired = true }; var packsOption = new Option("--packs", "When creating bundles, this specifies the packs to include, Example `pack1 pack2 pack3`. You may specify also as `--pack pack1 --pack2 --pack3`") { IsRequired = false, AllowMultipleArgumentsPerToken = true }; installCommand.Add(customBundleOption); @@ -48,7 +48,7 @@ public void Register(Command parentCommand) installCommand.Add(packsOption); runCommand.Add(installCommand); - + installCommand.SetHandler((basePath, automationType, customBundleOption, quickBundleOption, packs) => { Log.G().LogInformation("Executing install command..."); diff --git a/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs b/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs index d90ddcc..ee7eb07 100644 --- a/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs +++ b/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs @@ -15,16 +15,45 @@ public class InstallCommand : CommandTarget public bool CustomBundles { get; set; } public bool QuickBundles { get; set; } public string[] Packs { get; set; } + + void SetEnvironmentVariableMultiTarget(string name, string value) + { + Log.G().LogInformation($"Setting {name} to {value}..."); + + Environment.SetEnvironmentVariable(name, value); + + if (AutomationTypeHelper.AutomationTypeFromString(AutomationTarget) == AutomationType.ACTIONS) + { + string? githubEnvPath = Environment.GetEnvironmentVariable("GITHUB_ENV"); + try + { + if (File.Exists(githubEnvPath)) + { + File.AppendAllText(githubEnvPath, $"{name}={value}\n"); + } + else + { + throw new Exception("Could not find GITHUB_ENV file."); + } + } + catch (Exception) + { + Log.G().LogError($"Could not write to GITHUB_ENV file."); + throw; + } + } + } + public override void Run() { Log.G().LogInformation($"Running Install command"); // First, check if CodeQL is installed. var installation = CodeQLInstallation.LoadFromConfig(Base); - if(CustomBundles || QuickBundles) + if (CustomBundles || QuickBundles) { installation.EnableCustomCodeQLBundles = true; - if (Packs!=null && Packs.Length > 0) + if (Packs != null && Packs.Length > 0) { Log.G().LogInformation($"Overriding Packs on the command line. The following Packs will be packaged:"); installation.CodeQLPackConfiguration = Packs.Select(p => new CodeQLPackConfiguration() @@ -40,14 +69,12 @@ public override void Run() installation.LogPacksToBeBuilt(); - installation.QuickBundle = QuickBundles; } Log.G().LogInformation($"Checking for installation..."); - // if it is the case that it is installed but we are in custom bundle mode we RE install it. - + // If CodeQL is already installed, but custom bundles are enabled, reinstall CodeQL anyway to ensure use of the correct custom bundle. if (installation.IsInstalled() && !installation.EnableCustomCodeQLBundles) { Log.G().LogInformation($"CodeQL is already installed at that version. Please delete the installation directory to reinstall."); @@ -57,39 +84,19 @@ public override void Run() Log.G().LogInformation($"Installing CodeQL..."); installation.Install(); - // set the environment variable - Log.G().LogInformation($"Setting QLT_CODEQL_HOME to {installation.CodeQLHome}..."); - Log.G().LogInformation($"Setting QLT_CODEQL_PATH to {installation.CodeQLToolBinary}..."); + SetEnvironmentVariableMultiTarget("QLT_CODEQL_HOME", installation.CodeQLHome); + SetEnvironmentVariableMultiTarget("QLT_CODEQL_PATH", installation.CodeQLToolBinary); - Environment.SetEnvironmentVariable("QLT_CODEQL_HOME", installation.CodeQLHome); - Environment.SetEnvironmentVariable("QLT_CODEQL_PATH", installation.CodeQLToolBinary); if (CustomBundles || QuickBundles) { - Environment.SetEnvironmentVariable("QLT_CODEQL_BUNDLE_PATH", installation.CustomBundleOutputBundle); - } - - if (AutomationTypeHelper.AutomationTypeFromString(AutomationTarget) == AutomationType.ACTIONS) - { - if (Environment.GetEnvironmentVariable("GITHUB_ENV") != null && File.Exists(Environment.GetEnvironmentVariable("GITHUB_ENV"))) - { - - File.AppendAllText(Environment.GetEnvironmentVariable("GITHUB_ENV"), $"QLT_CODEQL_HOME={installation.CodeQLHome}" + "\n"); - File.AppendAllText(Environment.GetEnvironmentVariable("GITHUB_ENV"), $"QLT_CODEQL_PATH={installation.CodeQLToolBinary}" + "\n"); - if (CustomBundles || QuickBundles) - { - File.AppendAllText(Environment.GetEnvironmentVariable("GITHUB_ENV"), $"QLT_CODEQL_BUNDLE_PATH={installation.CustomBundleOutputBundle}" + "\n"); - } - } + SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH", installation.CustomBundleOutputBundleCurrentPlatform); + SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH_WIN64", installation.CustomBundleOutputBundleWindows); + SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH_OSX64", installation.CustomBundleOutputBundleOSX); + SetEnvironmentVariableMultiTarget("QLT_CODEQL_BUNDLE_PATH_LINUX64", installation.CustomBundleOutputBundleLinux); } - } - Log.G().LogInformation($"Done."); - - - - } } } diff --git a/src/CodeQLToolkit.Features/CodeQL/Lifecycle/Targets/GetVersionLifecycleTarget.cs b/src/CodeQLToolkit.Features/CodeQL/Lifecycle/Targets/GetVersionLifecycleTarget.cs index 2bd4823..250d5a5 100644 --- a/src/CodeQLToolkit.Features/CodeQL/Lifecycle/Targets/GetVersionLifecycleTarget.cs +++ b/src/CodeQLToolkit.Features/CodeQL/Lifecycle/Targets/GetVersionLifecycleTarget.cs @@ -9,7 +9,7 @@ namespace CodeQLToolkit.Features.CodeQL.Lifecycle.Targets { public class GetVersionLifecycleTarget : ILifecycleTarget { - + override public void Run() { Log.G().LogInformation("Running get command..."); @@ -19,11 +19,11 @@ override public void Run() Base = Base }; - if (!File.Exists(c.CodeQLConfigFilePath)) + if (!File.Exists(c.QLTConfigFilePath)) { - ProcessUtils.DieWithError($"Cannot read values from missing file {c.CodeQLConfigFilePath}"); + ProcessUtils.DieWithError($"Cannot read values from missing file {c.QLTConfigFilePath}"); } - + var config = c.FromFile(); // This should be updated so that we can pretty print all the various options: diff --git a/src/CodeQLToolkit.Features/Pack/Commands/Targets/HelloJeongsooCommandTarget.cs b/src/CodeQLToolkit.Features/Pack/Commands/Targets/HelloJeongsooCommandTarget.cs index 33b4420..0b34137 100644 --- a/src/CodeQLToolkit.Features/Pack/Commands/Targets/HelloJeongsooCommandTarget.cs +++ b/src/CodeQLToolkit.Features/Pack/Commands/Targets/HelloJeongsooCommandTarget.cs @@ -15,7 +15,8 @@ public class HelloJeongsooCommandTarget : CommandTarget public override void Run() { - for(int i = 0; i < Times; i++) { + for (int i = 0; i < Times; i++) + { Console.WriteLine($"Hello! My Base Target is: {Base}"); } @@ -25,9 +26,9 @@ public override void Run() Base = Base }; - if (!File.Exists(c.CodeQLConfigFilePath)) + if (!File.Exists(c.QLTConfigFilePath)) { - ProcessUtils.DieWithError($"Cannot read values from missing file {c.CodeQLConfigFilePath}"); + ProcessUtils.DieWithError($"Cannot read values from missing file {c.QLTConfigFilePath}"); } var config = c.FromFile(); diff --git a/src/CodeQLToolkit.Features/Pack/PackFeatureMain.cs b/src/CodeQLToolkit.Features/Pack/PackFeatureMain.cs index 9fa81ea..ad39124 100644 --- a/src/CodeQLToolkit.Features/Pack/PackFeatureMain.cs +++ b/src/CodeQLToolkit.Features/Pack/PackFeatureMain.cs @@ -23,7 +23,7 @@ private PackFeatureMain() { commandFeature = new PackCommandFeature(); } - public static PackFeatureMain Instance { get { return instance; } } + public static PackFeatureMain Instance => instance; public void Register(Command parentCommand) { diff --git a/src/CodeQLToolkit.Features/Query/QueryFeatureMain.cs b/src/CodeQLToolkit.Features/Query/QueryFeatureMain.cs index 6bc009e..ace37b8 100644 --- a/src/CodeQLToolkit.Features/Query/QueryFeatureMain.cs +++ b/src/CodeQLToolkit.Features/Query/QueryFeatureMain.cs @@ -18,7 +18,8 @@ public class QueryFeatureMain : IToolkitFeature readonly static QueryFeatureMain instance; - static QueryFeatureMain() { + static QueryFeatureMain() + { instance = new QueryFeatureMain(); } private QueryFeatureMain() @@ -28,7 +29,7 @@ private QueryFeatureMain() lifecycleFeature = new QueryLifecycleFeature(); } - public static QueryFeatureMain Instance { get { return instance; } } + public static QueryFeatureMain Instance => instance; public int Run() { @@ -46,7 +47,7 @@ public void Register(Command parentCommand) scaffoldFeature.Register(queryCommand); commandFeature.Register(queryCommand); lifecycleFeature.Register(queryCommand); - + } } } \ No newline at end of file diff --git a/src/CodeQLToolkit.Features/Query/Scaffolding/QueryScaffoldFeature.cs b/src/CodeQLToolkit.Features/Query/Scaffolding/QueryScaffoldFeature.cs index 6b16403..7c30af0 100644 --- a/src/CodeQLToolkit.Features/Query/Scaffolding/QueryScaffoldFeature.cs +++ b/src/CodeQLToolkit.Features/Query/Scaffolding/QueryScaffoldFeature.cs @@ -6,12 +6,13 @@ namespace CodeQLToolkit.Features.Query.Scaffolding { - public class QueryScaffoldFeature : FeatureBase, IToolkitScaffoldingFeature + public class QueryScaffoldFeature : FeatureBase, IToolkitScaffoldingFeature { - public QueryScaffoldFeature() { + public QueryScaffoldFeature() + { FeatureName = "Query"; } - public override LanguageType[] SupportedLangauges { get => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.JAVASCRIPT}; } + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.JAVASCRIPT }; public void Register(Command parentCommand) { @@ -26,9 +27,9 @@ public void Register(Command parentCommand) var createQueryPackOption = new Option("--create-query-pack", () => true, "Create a new query pack if none exists."); var overwriteExistingOption = new Option("--overwrite-existing", () => false, "Overwrite exiting files (if they exist)."); - var createTestsOption = new Option("--create-tests", ()=> true, "Create a new unit test for this query if it doesn't already exist."); + var createTestsOption = new Option("--create-tests", () => true, "Create a new unit test for this query if it doesn't already exist."); var queryNameOption = new Option("--query-name", "Name of the query. Note: Do not specify the `.ql` extension in naming your query.") { IsRequired = true }; - var queryLanguageOption = new Option("--language", $"The language to generate a query for.") { IsRequired = true} + var queryLanguageOption = new Option("--language", $"The language to generate a query for.") { IsRequired = true } .FromAmong(SupportedLangauges.Select(x => x.ToOptionString()).ToArray()); var queryPackOption = new Option("--pack", "The name of the query pack to place this query in.") { IsRequired = true }; var queryPackScopeOption = new Option("--scope", "The scope to use") { IsRequired = true }; @@ -62,8 +63,8 @@ public void Register(Command parentCommand) CreateTests = createTests, CreateQueryPack = createQueryPack, OverwriteExisting = overwriteExisting, - FeatureName = FeatureName - }.Run(); + FeatureName = FeatureName + }.Run(); }, createQueryPackOption, createTestsOption, queryNameOption, queryLanguageOption, queryPackOption, Globals.BasePathOption, overwriteExistingOption, queryPackScopeOption); } diff --git a/src/CodeQLToolkit.Features/Templates/Bundle/Actions/run-bundle-integration-tests.liquid b/src/CodeQLToolkit.Features/Templates/Bundle/Actions/run-bundle-integration-tests.liquid index 780cf2d..280c848 100644 --- a/src/CodeQLToolkit.Features/Templates/Bundle/Actions/run-bundle-integration-tests.liquid +++ b/src/CodeQLToolkit.Features/Templates/Bundle/Actions/run-bundle-integration-tests.liquid @@ -59,7 +59,6 @@ jobs: fi # ensure bundle runs - if ! qlt query run install-packs --use-bundle --base example/ ; then echo "Failed to install query packs with tool." exit 1 @@ -74,7 +73,6 @@ jobs: fi # ensure bundle runs - if ! qlt query run install-packs --use-bundle ; then echo "Failed to install query packs with tool." exit 1 @@ -87,13 +85,16 @@ jobs: echo "Checking Bundle Existence" ls -l ${{ env.QLT_CODEQL_HOME }}/../out/ - - name: Upload Bundle Used + - name: Upload Bundles uses: actions/upload-artifact@v2 with: - name: codeql-bundle.tar.gz + name: codeql-bundles path: | - ${{ env.QLT_CODEQL_BUNDLE_PATH }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_LINUX64 }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_WIN64 }} + ${{ env.QLT_CODEQL_BUNDLE_PATH_OSX64 }} if-no-files-found: error + compression-level: 0 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/src/CodeQLToolkit.Features/Test/Commands/Targets/Actions/GetMatrixCommandTarget.cs b/src/CodeQLToolkit.Features/Test/Commands/Targets/Actions/GetMatrixCommandTarget.cs index 666748d..52a7c9f 100644 --- a/src/CodeQLToolkit.Features/Test/Commands/Targets/Actions/GetMatrixCommandTarget.cs +++ b/src/CodeQLToolkit.Features/Test/Commands/Targets/Actions/GetMatrixCommandTarget.cs @@ -30,16 +30,16 @@ public override void Run() Base = Base }; - if (!File.Exists(c.CodeQLConfigFilePath)) + if (!File.Exists(c.QLTConfigFilePath)) { - ProcessUtils.DieWithError($"Cannot read values from missing file {c.CodeQLConfigFilePath}"); + ProcessUtils.DieWithError($"Cannot read values from missing file {c.QLTConfigFilePath}"); } var config = c.FromFile(); List configs = new List(); - foreach(var os in OSVersions) + foreach (var os in OSVersions) { Log.G().LogInformation($"Creating matrix for {os}"); @@ -66,7 +66,7 @@ public override void Run() Log.G().LogInformation($"Writing matrix output {matrixVariable} to {envFile}"); - File.AppendAllText(envFile, matrixVariable ); + File.AppendAllText(envFile, matrixVariable); Log.G().LogInformation($"Done."); diff --git a/src/CodeQLToolkit.Features/Test/Commands/TestCommandFeature.cs b/src/CodeQLToolkit.Features/Test/Commands/TestCommandFeature.cs index c3b661a..f05664f 100644 --- a/src/CodeQLToolkit.Features/Test/Commands/TestCommandFeature.cs +++ b/src/CodeQLToolkit.Features/Test/Commands/TestCommandFeature.cs @@ -14,7 +14,7 @@ namespace CodeQLToolkit.Features.Test.Commands { public class TestCommandFeature : FeatureBase, IToolkitLifecycleFeature { - public override LanguageType[] SupportedLangauges { get => new LanguageType[] { + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.CSHARP, @@ -22,8 +22,8 @@ public class TestCommandFeature : FeatureBase, IToolkitLifecycleFeature LanguageType.JAVASCRIPT, LanguageType.GO, LanguageType.RUBY, - LanguageType.PYTHON - }; } + LanguageType.PYTHON + }; public TestCommandFeature() { @@ -46,7 +46,7 @@ public void Register(Command parentCommand) // a command that runs the actual tests var unitTestsCommand = new Command("execute-unit-tests", "Runs unit tests within a repository based on the current configuration."); - + var numThreadsOption = new Option("--num-threads", () => 4, "The number of threads to use for runner. For best performance, do not exceed the number of physical cores on your system.") { IsRequired = true }; var workDirectoryOption = new Option("--work-dir", () => Path.GetTempPath(), "Where to place intermediate execution output files.") { IsRequired = true }; var languageOption = new Option("--language", $"The language to run tests for.") { IsRequired = true }.FromAmong(SupportedLangauges.Select(x => x.ToOptionString()).ToArray()); @@ -55,7 +55,7 @@ public void Register(Command parentCommand) //var stdLibIdentOption = new Option("--stdlib-ident", $"A string identifying the standard library used.") { IsRequired = true }; var extraCodeQLOptions = new Option("--codeql-args", $"Extra arguments to pass to CodeQL.") { IsRequired = false }; - unitTestsCommand.Add(numThreadsOption); + unitTestsCommand.Add(numThreadsOption); unitTestsCommand.Add(workDirectoryOption); unitTestsCommand.Add(languageOption); unitTestsCommand.Add(runnerOSOption); @@ -75,8 +75,9 @@ public void Register(Command parentCommand) runCommand.Add(getMatrixTestCommand); runCommand.Add(unitTestsCommand); runCommand.Add(validateUnitTestsCommand); - - getMatrixTestCommand.SetHandler((basePath, automationType, osVersions) => { + + getMatrixTestCommand.SetHandler((basePath, automationType, osVersions) => + { Log.G().LogInformation("Executing get-matrix command..."); @@ -94,7 +95,8 @@ public void Register(Command parentCommand) }, Globals.BasePathOption, Globals.AutomationTypeOption, matrixOSVersion); //stdLibIdent - unitTestsCommand.SetHandler((basePath, automationType, numThreads, workDirectory, language, runnerOS, extraArgs, useBundle) => { + unitTestsCommand.SetHandler((basePath, automationType, numThreads, workDirectory, language, runnerOS, extraArgs, useBundle) => + { Log.G().LogInformation("Executing execute-unit-tests command..."); @@ -110,9 +112,9 @@ public void Register(Command parentCommand) Base = basePath }; - if (!File.Exists(c.CodeQLConfigFilePath)) + if (!File.Exists(c.QLTConfigFilePath)) { - ProcessUtils.DieWithError($"Cannot read values from missing file {c.CodeQLConfigFilePath}"); + ProcessUtils.DieWithError($"Cannot read values from missing file {c.QLTConfigFilePath}"); } var config = c.FromFile(); @@ -129,12 +131,12 @@ public void Register(Command parentCommand) featureTarget.Run(); - }, Globals.BasePathOption, - Globals.AutomationTypeOption, + }, Globals.BasePathOption, + Globals.AutomationTypeOption, numThreadsOption, - workDirectoryOption, - languageOption, - runnerOSOption, + workDirectoryOption, + languageOption, + runnerOSOption, extraCodeQLOptions, Globals.UseBundle ); diff --git a/src/CodeQLToolkit.Features/Test/Lifecycle/TestLifecycleFeature.cs b/src/CodeQLToolkit.Features/Test/Lifecycle/TestLifecycleFeature.cs index 643937d..0745421 100644 --- a/src/CodeQLToolkit.Features/Test/Lifecycle/TestLifecycleFeature.cs +++ b/src/CodeQLToolkit.Features/Test/Lifecycle/TestLifecycleFeature.cs @@ -8,14 +8,12 @@ namespace CodeQLToolkit.Features.Test.Lifecycle { public class TestLifecycleFeature : FeatureBase, IToolkitLifecycleFeature { - public TestLifecycleFeature() + public TestLifecycleFeature() { FeatureName = "Test"; } - public override LanguageType[] SupportedLangauges - { - get => new LanguageType[] { + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.CSHARP, @@ -25,7 +23,6 @@ public override LanguageType[] SupportedLangauges LanguageType.RUBY, LanguageType.PYTHON }; - } public void Register(Command parentCommand) { @@ -50,7 +47,7 @@ public void Register(Command parentCommand) parentCommand.Add(initCommand); - + initCommand.SetHandler((devMode, basePath, automationType, overwriteExisting, numThreads, useRunner, language, branch) => { Log.G().LogInformation("Executing init command..."); @@ -62,14 +59,14 @@ public void Register(Command parentCommand) // setup common params featureTarget.FeatureName = FeatureName; - featureTarget.Base = basePath; + featureTarget.Base = basePath; featureTarget.OverwriteExisting = overwriteExisting; featureTarget.NumThreads = numThreads; - featureTarget.UseRunner = useRunner; + featureTarget.UseRunner = useRunner; featureTarget.Language = language; //featureTarget.ExtraArgs = extraArgs; featureTarget.DevMode = devMode; - featureTarget.Branch = branch; + featureTarget.Branch = branch; featureTarget.Run(); }, Globals.Development, Globals.BasePathOption, Globals.AutomationTypeOption, overwriteExistingOption, numThreadsOption, useRunnerOption, languageOption, branchOption); diff --git a/src/CodeQLToolkit.Features/Test/TestFeatureMain.cs b/src/CodeQLToolkit.Features/Test/TestFeatureMain.cs index 70f45eb..3db642a 100644 --- a/src/CodeQLToolkit.Features/Test/TestFeatureMain.cs +++ b/src/CodeQLToolkit.Features/Test/TestFeatureMain.cs @@ -4,7 +4,7 @@ using System.CommandLine; namespace CodeQLToolkit.Features.Test -{ +{ public class TestFeatureMain : IToolkitFeature { @@ -20,10 +20,10 @@ static TestFeatureMain() private TestFeatureMain() { lifecycleFeature = new TestLifecycleFeature(); - commandFeature = new TestCommandFeature(); + commandFeature = new TestCommandFeature(); } - public static TestFeatureMain Instance { get { return instance; } } - + public static TestFeatureMain Instance => instance; + public void Register(Command parentCommand) { var testCommand = new Command("test", "Features related to the running and processing of CodeQL Unit Tests."); diff --git a/src/CodeQLToolkit.Features/Validation/Commands/ValidationCommandFeature.cs b/src/CodeQLToolkit.Features/Validation/Commands/ValidationCommandFeature.cs index 31eae45..9dcf2f8 100644 --- a/src/CodeQLToolkit.Features/Validation/Commands/ValidationCommandFeature.cs +++ b/src/CodeQLToolkit.Features/Validation/Commands/ValidationCommandFeature.cs @@ -6,7 +6,7 @@ namespace CodeQLToolkit.Features.Test.Commands { public class ValidationCommandFeature : FeatureBase, IToolkitLifecycleFeature { - public override LanguageType[] SupportedLangauges { get => new LanguageType[] { + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.CSHARP, @@ -14,8 +14,8 @@ public class ValidationCommandFeature : FeatureBase, IToolkitLifecycleFeature LanguageType.JAVASCRIPT, LanguageType.GO, LanguageType.RUBY, - LanguageType.PYTHON - }; } + LanguageType.PYTHON + }; public ValidationCommandFeature() { @@ -49,10 +49,10 @@ public void Register(Command parentCommand) { Base = basePath, Language = language, - PrettyPrint = prettyPrint, + PrettyPrint = prettyPrint, UseBundle = useBundle }.Run(); - + }, languageOption, Globals.BasePathOption, prettyPrintOption, Globals.UseBundle); } diff --git a/src/CodeQLToolkit.Features/Validation/Lifecycle/ValidationLifecycleFeature.cs b/src/CodeQLToolkit.Features/Validation/Lifecycle/ValidationLifecycleFeature.cs index 101e826..0c2600a 100644 --- a/src/CodeQLToolkit.Features/Validation/Lifecycle/ValidationLifecycleFeature.cs +++ b/src/CodeQLToolkit.Features/Validation/Lifecycle/ValidationLifecycleFeature.cs @@ -16,9 +16,7 @@ public ValidationLifecycleFeature() FeatureName = "Validation"; } - public override LanguageType[] SupportedLangauges - { - get => new LanguageType[] { + public override LanguageType[] SupportedLangauges => new LanguageType[] { LanguageType.C, LanguageType.CPP, LanguageType.CSHARP, @@ -28,7 +26,6 @@ public override LanguageType[] SupportedLangauges LanguageType.RUBY, LanguageType.PYTHON }; - } public void Register(Command parentCommand) { @@ -64,7 +61,7 @@ public void Register(Command parentCommand) featureTarget.OverwriteExisting = overwriteExisting; featureTarget.Language = language; featureTarget.DevMode = devMode; - featureTarget.Branch = branch; + featureTarget.Branch = branch; featureTarget.Run(); }, Globals.Development, Globals.BasePathOption, Globals.AutomationTypeOption, overwriteExistingOption, languageOption, useRunnerOption, branchOption); diff --git a/src/CodeQLToolkit.Features/Validation/ValidationFeatureMain.cs b/src/CodeQLToolkit.Features/Validation/ValidationFeatureMain.cs index 27e24e4..76702d7 100644 --- a/src/CodeQLToolkit.Features/Validation/ValidationFeatureMain.cs +++ b/src/CodeQLToolkit.Features/Validation/ValidationFeatureMain.cs @@ -18,15 +18,15 @@ static ValidationFeatureMain() private ValidationFeatureMain() { - commandFeature = new ValidationCommandFeature(); + commandFeature = new ValidationCommandFeature(); validationLifecycleFeature = new ValidationLifecycleFeature(); } - public static ValidationFeatureMain Instance { get { return instance; } } - + public static ValidationFeatureMain Instance => instance; + public void Register(Command parentCommand) { var validationCommand = new Command("validation", "Features related to the validation of CodeQL Development Repositories."); - parentCommand.Add(validationCommand); + parentCommand.Add(validationCommand); Log.G().LogInformation("Registering command submodule."); commandFeature.Register(validationCommand); diff --git a/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs b/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs index 01c443c..9f8ba89 100644 --- a/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs +++ b/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs @@ -8,8 +8,6 @@ using System.Diagnostics; using System; - - namespace CodeQLToolkit.Shared.CodeQL { public class CodeQLInstallation @@ -17,13 +15,14 @@ public class CodeQLInstallation public string CLIVersion { get; set; } public string StandardLibraryVersion { get; set; } - public string CodeQLConfiguration { get; set; } + public string CodeScanningConfig { get; set; } + public string QLTConfigFilePath { get; private set; } public string CLIBundle { get; set; } public string StandardLibraryIdent { get; set; } public bool EnableCustomCodeQLBundles { get; set; } public CodeQLPackConfiguration[] CodeQLPackConfiguration { get; set; } - public bool QuickBundle { get; set; } + public bool QuickBundle { get; set; } public string Base { get; set; } public static CodeQLInstallation LoadFromConfig(string Base) @@ -33,14 +32,13 @@ public static CodeQLInstallation LoadFromConfig(string Base) Base = Base }; - return LoadFromConfig(c.FromFile()); + return LoadFromConfig(c.FromFile()); } public static CodeQLInstallation LoadFromConfig(QLTConfig c) { var config = c.FromFile(); - return new CodeQLInstallation { CLIVersion = config.CodeQLCLI, @@ -49,24 +47,23 @@ public static CodeQLInstallation LoadFromConfig(QLTConfig c) StandardLibraryVersion = config.CodeQLStandardLibrary, CodeQLPackConfiguration = config.CodeQLPackConfiguration, Base = config.Base, - CodeQLConfiguration = config.CodeQLConfiguration + CodeScanningConfig = config.CodeQLConfiguration, + QLTConfigFilePath = config.QLTConfigFilePath }; - - } public void LogPacksToBeBuilt() { - if(CodeQLPackConfiguration != null) + if (CodeQLPackConfiguration != null) { - foreach(var p in CodeQLPackConfiguration) + foreach (var p in CodeQLPackConfiguration) { Log.G().LogInformation($"Pack: {p}"); } } } - public ArtifactKind Kind + public ArtifactKind Kind { get { @@ -83,8 +80,6 @@ public ArtifactKind Kind } } - - public string PlatformID { get @@ -108,16 +103,9 @@ public string PlatformID } } - public string PlatformExtension - { - get - { - // for now they are all zips, - return "zip"; - } - } + public string PlatformExtension => "zip"; + - public void Install() { // each time we download the file; however, @@ -126,71 +114,44 @@ public void Install() // .qlt/repo/bundle/ident // https://github.com/github/codeql-cli-binaries/releases/download/v2.16.0/codeql-linux64.zip - + // workout a destination directory for this - if(Kind == ArtifactKind.CUSTOM_BUNDLE) + if (Kind == ArtifactKind.CUSTOM_BUNDLE) { CustomBundleInstall(); - }else if(Kind == ArtifactKind.BUNDLE) - { - BundleInstall(); - }else - { - PackageInstall(); } - - - - - } - - public string CustomBundleOutputBundle - { - get + else if (Kind == ArtifactKind.BUNDLE) { - return Path.Combine(CustomBundleOutputDirectory, "codeql-bundle.tar.gz"); + BundleInstall(); } - } - public string CustomBundleOutputDirectory - { - get + else { - return Path.Combine(InstallationDirectory, "out"); + PackageInstall(); } } - public string StdLibDirectory - { - get - { - return Path.Combine(InstallationDirectory, "codeql-stdlib"); + public string CustomBundleOutputBundleCurrentPlatform => Path.Combine(CustomBundleOutputDirectory, $"codeql-bundle-{PlatformID}.tar.gz"); + public string CustomBundleOutputBundleLinux => Path.Combine(CustomBundleOutputDirectory, "codeql-bundle-linux64.tar.gz"); + public string CustomBundleOutputBundleWindows => Path.Combine(CustomBundleOutputDirectory, "codeql-bundle-win64.tar.gz"); + public string CustomBundleOutputBundleOSX => Path.Combine(CustomBundleOutputDirectory, "codeql-bundle-osx64.tar.gz"); - } - } + public string CustomBundleOutputDirectory => Path.Combine(InstallationDirectory, "out"); - public string CodeQLDirectory - { - get - { - return Path.Combine(InstallationDirectory, "codeql"); + public string StdLibDirectory => Path.Combine(InstallationDirectory, "codeql-stdlib"); + public string CodeQLDirectory => Path.Combine(InstallationDirectory, "codeql"); - } - } private void PackageInstall() { Log.G().LogInformation($"Begin Installation "); - Log.G().LogInformation($"Requested CLI Version {CLIVersion}, Standard Library Ident: {StandardLibraryIdent}"); + Log.G().LogInformation($"Requested CLI Version {CLIVersion}, Standard Library Ident: {StandardLibraryIdent}"); Log.G().LogInformation($"Create installation directory {InstallationDirectory}"); - Directory.CreateDirectory(InstallationDirectory); - Log.G().LogInformation($"Download CodeQL CLI..."); - var downloadFile = $"codeql-{PlatformID}.{PlatformExtension}"; // first, download the cli. @@ -217,20 +178,20 @@ private void PackageInstall() using (var repo = new Repository(repoPath)) { var tag = repo.Tags[$"refs/tags/{StandardLibraryVersion}"]; - + if (tag == null) { Log.G().LogInformation($"Unknown standard library version: {StandardLibraryVersion}"); throw new Exception($"Unknown standard library version: {StandardLibraryVersion}"); - } - + } + Branch b = Commands.Checkout(repo, $"refs/tags/{StandardLibraryVersion}"); } } private void CustomBundleInstall() { - Log.G().LogInformation($"Begin Installation "); + Log.G().LogInformation($"Begin Installation"); Log.G().LogInformation($"Requested Base Bundle Version {CLIBundle}"); Log.G().LogInformation($"Create installation directory {InstallationDirectory}"); @@ -239,35 +200,33 @@ private void CustomBundleInstall() Directory.CreateDirectory(InstallationDirectory); } - Log.G().LogInformation($"Download CodeQL Bundle Base..."); + // Download the platform-independent bundle. + Log.G().LogInformation($"Downloading CodeQL base bundle..."); - var downloadFile = $"codeql-bundle-{PlatformID}.tar.gz"; - var customBundleSource = Path.Combine(InstallationDirectory, downloadFile); - - Log.G().LogInformation($"Checking if a existing source bundle is present..."); + var downloadFile = $"codeql-bundle.tar.gz"; + var customBundlePath = Path.Combine(InstallationDirectory, downloadFile); - if (File.Exists(customBundleSource)) + Log.G().LogInformation($"Checking if existing source bundle {downloadFile} is present..."); + + if (File.Exists(customBundlePath)) { Log.G().LogInformation($"Bundle exists, will skip download"); } else { - using (var client = new WebClient()) - { - string uri = $"https://github.com/github/codeql-action/releases/download/{CLIBundle}/{downloadFile}"; - Log.G().LogInformation($"Remote URL: {uri}..."); - - client.DownloadFile(uri, customBundleSource); - } + using var client = new WebClient(); + string uri = $"https://github.com/github/codeql-action/releases/download/{CLIBundle}/{downloadFile}"; + Log.G().LogInformation($"Downloading platform-independent bundle from remote URL: {uri}..."); + client.DownloadFile(uri, customBundlePath); } - // next, we create the output directory that will contain the bundle. - // if it exists, we remove it + // Create the output directory that will contain the custom bundles. + // If it already exists, delete it and recreate it. Log.G().LogInformation($"Checking for custom bundle output directory..."); if (Directory.Exists(CustomBundleOutputDirectory)) { - Log.G().LogInformation($"Exists. Will remove."); + Log.G().LogInformation($"Exists. Removing..."); Directory.Delete(CustomBundleOutputDirectory, true); Directory.CreateDirectory(CustomBundleOutputDirectory); } @@ -278,20 +237,18 @@ private void CustomBundleInstall() var workingDirectory = Path.GetFullPath(Base); - if(CodeQLPackConfiguration == null || CodeQLPackConfiguration.Length == 0) + if (CodeQLPackConfiguration == null || CodeQLPackConfiguration.Length == 0) { throw new Exception("No packs are set to be exported. Please add at least one pack to export in your `qlt.conf.json` file under the property `ExportedCustomizationPacks`."); } - Log.G().LogInformation($"Building custom bundle. This may take a while..."); - - var packsToExport = CodeQLPackConfiguration.Where(p => p.Bundle == true).Select(p => p.Name).ToArray(); + Log.G().LogInformation($"Building custom bundles. This may take a while..."); + var packsToExport = CodeQLPackConfiguration.Where(p => p.Bundle == true).Select(p => p.Name).ToArray(); var packs = string.Join(" ", packsToExport); - // next, we run the bundling tool. - // typical command line: - // codeql_bundle -b .\scratch\codeql-bundle-win64.tar.gz -o scratch\out -w .\tests\workspace\ --help - var bundleArgs = $"--log DEBUG -b {customBundleSource} -o {CustomBundleOutputDirectory} -w {workingDirectory} {packs}"; + + // Run the bundling tool to create the platform-specific custom bundles from the platform-independent bundle + var bundleArgs = $"--log DEBUG -a qlt.conf.json -p win64 -p osx64 -p linux64 -b {customBundlePath} -o {CustomBundleOutputDirectory} -w {workingDirectory} {packs}"; if (QuickBundle) { @@ -299,13 +256,13 @@ private void CustomBundleInstall() bundleArgs = $"-nc {bundleArgs}"; } - if(CodeQLConfiguration!=null && CodeQLConfiguration.Length > 0) + if (CodeScanningConfig != null && CodeScanningConfig.Length > 0) { Log.G().LogInformation($"Note: Attempting to include default code scanning configuration ..."); - if (File.Exists(Path.Combine(Base, CodeQLConfiguration))) + if (File.Exists(Path.Combine(Base, CodeScanningConfig))) { - bundleArgs = $"-c \"{Path.Combine(Base, CodeQLConfiguration)}\" {bundleArgs}"; + bundleArgs = $"-c \"{Path.Combine(Base, CodeScanningConfig)}\" {bundleArgs}"; } } @@ -330,7 +287,6 @@ private void CustomBundleInstall() } process.Start(); - process.WaitForExit(); if (process.ExitCode != 0) @@ -340,12 +296,8 @@ private void CustomBundleInstall() } } - // once that is complete we expand the archive that is generated. - // it will be `codeql-bundle.tar.gz`. - // - // we will extract it to `InstallationDirectory` since `codeql` will be created by the extraction. - - Log.G().LogInformation($"Done. Checking for existance of {CodeQLDirectory}"); + // Expand the correct platform-specific archive that is generated by the bundle command to `InstallationDirectory` + Log.G().LogInformation($"Checking for existence of {CodeQLDirectory}"); if (Directory.Exists(CodeQLDirectory)) { @@ -353,10 +305,8 @@ private void CustomBundleInstall() Directory.Delete(CodeQLDirectory, true); } - // Using SharpCompress, extract the tar.gz bundle to the `InstallationDirectory`. Log.G().LogInformation($"Extracting bundle to {InstallationDirectory}..."); - using (Process process = new Process()) { process.StartInfo.FileName = ToolUtil.GetCommand("tar"); @@ -364,10 +314,9 @@ private void CustomBundleInstall() process.StartInfo.UseShellExecute = false; process.StartInfo.RedirectStandardOutput = false; process.StartInfo.RedirectStandardError = false; - process.StartInfo.Arguments = $"-zxf {CustomBundleOutputBundle} "; + process.StartInfo.Arguments = $"-zxf {CustomBundleOutputBundleCurrentPlatform} "; process.Start(); - process.WaitForExit(); if (process.ExitCode != 0) @@ -390,11 +339,11 @@ private string GetIdentForPackage(ArtifactKind k) { if (k == ArtifactKind.PACKAGE) { - var ident = String.Join("", "codeql-cli-" + CLIVersion, "#standard-library-ident-" ,StandardLibraryIdent); + var ident = String.Join("", "codeql-cli-" + CLIVersion, "#standard-library-ident-", StandardLibraryIdent); return StringUtils.CreateMD5(FileUtils.SanitizeFilename(ident)).ToLower(); } - if( k == ArtifactKind.CUSTOM_BUNDLE) + if (k == ArtifactKind.CUSTOM_BUNDLE) { var ident = String.Join("", "codeql-bundle-" + CLIBundle); return StringUtils.CreateMD5(FileUtils.SanitizeFilename(ident)).ToLower(); @@ -427,7 +376,7 @@ public bool IsInstalled() { Log.G().LogInformation($"Requested CLI Version {CLIVersion} with Standard Library Ident: {StandardLibraryIdent}"); } - + Log.G().LogInformation($"Checking for existance of required directories..."); if (!Directory.Exists(InstallationDirectory)) @@ -456,12 +405,7 @@ public bool IsInstalled() return true; } - public string InstallationDirectory - { - get { - return GetInstallationDirectory(Kind); - } - } + public string InstallationDirectory => GetInstallationDirectory(Kind); public string GetInstallationDirectory(ArtifactKind k) { @@ -469,21 +413,18 @@ public string GetInstallationDirectory(ArtifactKind k) } - public string CodeQLHome { - get { - return CodeQLDirectory; - } - } + public string CodeQLHome => CodeQLDirectory; - public string CodeQLToolBinary { - get { + public string CodeQLToolBinary + { + get + { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { return Path.Combine(CodeQLDirectory, "codeql.exe"); } return Path.Combine(CodeQLDirectory, "codeql"); - } + } } - } } diff --git a/src/CodeQLToolkit.Shared/CodeQL/InstallationRepository.cs b/src/CodeQLToolkit.Shared/CodeQL/InstallationRepository.cs index 1c77a4f..51e7e98 100644 --- a/src/CodeQLToolkit.Shared/CodeQL/InstallationRepository.cs +++ b/src/CodeQLToolkit.Shared/CodeQL/InstallationRepository.cs @@ -14,46 +14,23 @@ public class InstallationRepository readonly static string BUNDLE_DIRECTORY = "bundle"; readonly static string CUSTOM_BUNDLE_DIRECTORY = "custom-bundle"; - public static string GetLocation - { - get - { - return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".qlt"); - } - } + public static string GetLocation => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".qlt"); - public static string PackageLocation - { - get - { - return Path.Combine(GetLocation, PAKCAGE_DIRECTORY); - } - } + public static string PackageLocation => Path.Combine(GetLocation, PAKCAGE_DIRECTORY); - public static string BundleLocation - { - get - { - return Path.Combine(GetLocation, BUNDLE_DIRECTORY); - } - } + public static string BundleLocation => Path.Combine(GetLocation, BUNDLE_DIRECTORY); - public static string CustomBundleLocation - { - get - { - return Path.Combine(GetLocation, CUSTOM_BUNDLE_DIRECTORY); - } - } + public static string CustomBundleLocation => Path.Combine(GetLocation, CUSTOM_BUNDLE_DIRECTORY); public static string DirectoryForVersion(ArtifactKind kind, string version) { - if(kind == ArtifactKind.PACKAGE) + if (kind == ArtifactKind.PACKAGE) { return Path.Combine(PackageLocation, version); } - else if (kind == ArtifactKind.BUNDLE) { + else if (kind == ArtifactKind.BUNDLE) + { return Path.Combine(BundleLocation, version); } else diff --git a/src/CodeQLToolkit.Shared/CodeQL/RESOLUTION.md b/src/CodeQLToolkit.Shared/CodeQL/RESOLUTION.md index b8fef98..0d1b4ca 100644 --- a/src/CodeQLToolkit.Shared/CodeQL/RESOLUTION.md +++ b/src/CodeQLToolkit.Shared/CodeQL/RESOLUTION.md @@ -27,7 +27,7 @@ list of packs as the packs to include in the bundle. Note this will compile all of the queries and may take a long time. In this case, you may create a bundle as follows: ``` -qlt codeql install --quick-bundle [--packs pack1 pack2 pack3] +qlt codeql install --quick-bundle [--packs pack1 pack2 pack3] ``` Which will not compile the packs and queries. @@ -47,10 +47,17 @@ For a bundle installation the mapping is as follows: - `CodeQLCLIBundle` - The bundle downloaded from `github/codeql-action/releases` to base the bundle on. -In all cases, at the end of the execution two to three environment variables are set: -- `QLT_CODEQL_PATH` - The path to the CodeQL binary. -- `QLT_CODEQL_HOME` - The root installation of CodeQL -- `QLT_CODEQL_BUNDLE_PATH` - The path to the bundle created by QLT. +In all cases, two environment variables are set after a run: +- `QLT_CODEQL_PATH` - The path to the CodeQL binary. (Always set) +- `QLT_CODEQL_HOME` - The root installation of CodeQL. (Always set) + +When using custom bundles, four additional environmental variables are set after a run: +- `QLT_CODEQL_BUNDLE_PATH` - The path to the current platform bundle created by QLT. (Set when using custom bundles) +- `QLT_CODEQL_BUNDLE_PATH_WIN64` - The path to the Windows bundle created by QLT. (Set when using custom bundles) +- `QLT_CODEQL_BUNDLE_PATH_LINUX64` - The path to the Linux bundle created by QLT. (Set when using custom bundles) +- `QLT_CODEQL_BUNDLE_PATH_OSX64` - The path to the MacOS bundle created by QLT. (Set when using custom bundles) + +The environmental variable `QLT_CODE_BUNDLE_PATH` will map to one of the three other bundle variables. ## Idents within the Installation Directory diff --git a/src/CodeQLToolkit.Shared/Options/Globals.cs b/src/CodeQLToolkit.Shared/Options/Globals.cs index 8be4a03..ad30aea 100644 --- a/src/CodeQLToolkit.Shared/Options/Globals.cs +++ b/src/CodeQLToolkit.Shared/Options/Globals.cs @@ -6,27 +6,32 @@ using System.Text; using System.Threading.Tasks; using CodeQLToolkit.Shared.Types; - + namespace CodeQLToolkit.Shared.Options { public class Globals { - public static string[] SupportedAutomationTypes { get => new string[] { "actions" }; } + public static string[] SupportedAutomationTypes => new string[] { "actions" }; - public static Option BasePathOption { get; } = new Option("--base", () => { + public static Option BasePathOption { get; } = new Option("--base", () => + { return Directory.GetCurrentDirectory(); }, "The base path to find the query repository."); - public static Option AutomationTypeOption { get; } = new Option("--automation-type", () => { + public static Option AutomationTypeOption { get; } = new Option("--automation-type", () => + { return "actions"; - }, "The base path to find the query repository.") { IsRequired = true }.FromAmong(SupportedAutomationTypes); + }, "The base path to find the query repository.") + { IsRequired = true }.FromAmong(SupportedAutomationTypes); - public static Option Development { get; } = new Option("--development", () => { + public static Option Development { get; } = new Option("--development", () => + { return false; }, "Turns on development mode which enables special features used in the development of QLT.") { IsRequired = true }.FromAmong(SupportedAutomationTypes); - public static Option UseBundle { get; } = new Option("--use-bundle", () => { + public static Option UseBundle { get; } = new Option("--use-bundle", () => + { return false; }, "Switching QLT from using the distribution versions of CodeQL to using a Custom Bundle.") { IsRequired = true }.FromAmong(SupportedAutomationTypes); diff --git a/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs b/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs index c9663d4..48eeb35 100644 --- a/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs +++ b/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs @@ -11,7 +11,7 @@ public class CodeQLPackConfiguration { public string Name { get; set; } public bool Bundle { get; set; } - public bool Publish { get; set;} + public bool Publish { get; set; } public bool ReferencesBundle { get; set; } } @@ -24,37 +24,32 @@ public class QLTConfig public string CodeQLConfiguration { get; set; } public CodeQLPackConfiguration[] CodeQLPackConfiguration { get; set; } - - public string CodeQLStandardLibraryIdent { - get { + + public string CodeQLStandardLibraryIdent + { + get + { if (CodeQLStandardLibrary != null) { return CodeQLStandardLibrary.Replace("/", "_"); } return CodeQLStandardLibrary; - } + } } [JsonIgnore] - public string CodeQLConfigurationPath { get { return Path.Combine(Base, CodeQLConfiguration); } } + public string CodeQLConfigurationPath => Path.Combine(Base, CodeQLConfiguration); [JsonIgnore] public string Base { get; set; } [JsonIgnore] - public string CodeQLConfigFilePath - { - get - { - return Path.Combine(Base, "qlt.conf.json"); - } - } + public string QLTConfigFilePath => Path.Combine(Base, "qlt.conf.json"); - public QLTConfig FromFile() { - var data = File.ReadAllText(CodeQLConfigFilePath); + public QLTConfig FromFile() + { + var data = File.ReadAllText(QLTConfigFilePath); QLTConfig c = JsonConvert.DeserializeObject(data); - - c.Base = Base; return c; } @@ -62,7 +57,7 @@ public QLTConfig FromFile() { public void ToFile() { var data = JsonConvert.SerializeObject(this, Formatting.Indented); - File.WriteAllText(CodeQLConfigFilePath, data); + File.WriteAllText(QLTConfigFilePath, data); } public static QLTConfig? LoadFromFile(string baseDir) @@ -71,9 +66,9 @@ public void ToFile() { Base = baseDir }; - - if (File.Exists(config.CodeQLConfigFilePath)) + + if (File.Exists(config.QLTConfigFilePath)) { return config.FromFile(); } diff --git a/src/CodeQLToolkit.Shared/Utils/Query.cs b/src/CodeQLToolkit.Shared/Utils/Query.cs index 9f035a3..53f12e4 100644 --- a/src/CodeQLToolkit.Shared/Utils/Query.cs +++ b/src/CodeQLToolkit.Shared/Utils/Query.cs @@ -1,4 +1,4 @@ - using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -14,97 +14,28 @@ public class Query public string Scope { get; set; } public string Name { get; set; } - public string QuerySrcDir - { - get - { - return Path.Combine(Base, Language.ToDirectory(), QueryPackName, "src"); - } - } + public string QuerySrcDir => Path.Combine(Base, Language.ToDirectory(), QueryPackName, "src"); - public string QueryFileDir - { - get - { - return Path.Combine(QuerySrcDir, Name); - } - } + public string QueryFileDir => Path.Combine(QuerySrcDir, Name); - public string QueryFilePath - { - get - { - return Path.Combine(QueryFileDir, $"{Name}.ql"); - } - } + public string QueryFilePath => Path.Combine(QueryFileDir, $"{Name}.ql"); - public string QueryPackPath - { - get - { - return Path.Combine(QuerySrcDir, "qlpack.yml"); - - } - } + public string QueryPackPath => Path.Combine(QuerySrcDir, "qlpack.yml"); // - public string QueryTestDir - { - get - { - return Path.Combine(Base, Language.ToDirectory(), QueryPackName, "test"); - } - } + public string QueryTestDir => Path.Combine(Base, Language.ToDirectory(), QueryPackName, "test"); - public string QueryFileTestDir - { - get - { - return Path.Combine(QueryTestDir, Name); - } - } + public string QueryFileTestDir => Path.Combine(QueryTestDir, Name); - public string QueryFileTestPath - { - get - { - return Path.Combine(QueryFileTestDir, $"{Name}.{Language.ToExtension()}"); - } - } + public string QueryFileTestPath => Path.Combine(QueryFileTestDir, $"{Name}.{Language.ToExtension()}"); - public string QueryFileQLRefPath - { - get - { - return Path.Combine(QueryFileTestDir, $"{Name}.qlref"); - } - } + public string QueryFileQLRefPath => Path.Combine(QueryFileTestDir, $"{Name}.qlref"); - public string QueryPackTestPath - { - get - { - return Path.Combine(QueryTestDir, "qlpack.yml"); + public string QueryPackTestPath => Path.Combine(QueryTestDir, "qlpack.yml"); - } - } + public string QueryTestExpectedFile => Path.Combine(QueryFileTestDir, $"{Name}.expected"); - public string QueryTestExpectedFile - { - get - { - return Path.Combine(QueryFileTestDir, $"{Name}.expected"); - - } - } - - public string QueryTestPackName - { - get - { - return $"{QueryPackName}-tests"; - } - } + public string QueryTestPackName => $"{QueryPackName}-tests"; public string GetLanguageImportForLangauge() { diff --git a/src/CodeQLToolkit.Shared/Utils/ToolUtil.cs b/src/CodeQLToolkit.Shared/Utils/ToolUtil.cs index 28fd017..beb39f2 100644 --- a/src/CodeQLToolkit.Shared/Utils/ToolUtil.cs +++ b/src/CodeQLToolkit.Shared/Utils/ToolUtil.cs @@ -9,14 +9,8 @@ namespace CodeQLToolkit.Shared.Utils { public class ToolUtil { - public static string ToolRoot - { - get - { - return Path.Combine(Utils.FileUtils.GetExecutingDirectory().FullName, "tools"); - } - } - + public static string ToolRoot => Path.Combine(Utils.FileUtils.GetExecutingDirectory().FullName, "tools"); + public static string ExecutableExtensionForPlatform { get