adding metadata validation

src/CodeQLToolkit.Features/CodeQLToolkit.Features.csproj

@@ -12,9 +12,8 @@
 
   <ItemGroup>
     <Folder Include="CodeQL\" />
-    <Folder Include="Validation\" />
     <Folder Include="Pack\" />
-    <Folder Include="Validation\Commands\Targets\" />
+    <Folder Include="Templates\Validation\Actions\" />
   </ItemGroup>
 
   <ItemGroup>
@@ -72,6 +71,9 @@
     <None Update="Templates\Test\Actions\run-unit-tests.liquid">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="Templates\Validation\Actions\validate-query-metadata.liquid">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>

src/CodeQLToolkit.Features/Templates/Validation/Actions/validate-query-metadata.liquid

@@ -0,0 +1,86 @@
+name: ⚙️ CodeQL - Validate Queries ({{language}})
+{% raw %}
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  create-matrix:
+    name: Create CodeQL Test Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.export-test-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install QLT
+        id: install-qlt
+        uses: ./.github/actions/install-qlt
+        with:
+          qlt-version: 'latest'
+          add-to-path: true
+{% endraw %}
+      - name: Export test matrix
+        id: export-test-matrix
+        run: |
+          qlt test run get-matrix --os-version {{ use_runner }}
+{% raw %}
+  validate-queries:
+    name: Validate Queries
+    needs: create-matrix
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.create-matrix.outputs.matrix) }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install QLT
+        uses: ./.github/actions/install-qlt
+        with:
+          qlt-version: 'latest'
+          add-to-path: true
+      
+      - name: Install CodeQL
+        uses: ./.github/actions/install-codeql
+        with:
+          codeql-cli-version: ${{ matrix.codeql_cli }}
+          codeql-stdlib-version: ${{ matrix.codeql_standard_library }}
+          add-to-path: true
+
+      - name: Verify Versions of Tooling
+        shell: bash
+        run: |
+          echo "CodeQL Home: ${{ steps.install-codeql.outputs.codeql-home }}"
+          echo -e "Checking CodeQL Version:"
+          codeql --version 
+          
+          echo -e "Checking QLT Version:"
+          echo "QLT Home: ${{ steps.install-qlt.outputs.qlt-home }}"
+          qlt version
+
+      - name: Install QL Packs
+        shell: bash
+        run: |
+          qlt query run install-packs
+
+      - name: Run validation tests
+        shell: bash
+        run: >
+{% endraw %}
+          # run a copy for pretty printing
+          qlt validation run check-queries --pretty-print 
+          --language {{ language }} 
+
+          # run this version to influence the outcome of the run. 
+          qlt validation run check-queries
+          --language {{ language }} 

src/CodeQLToolkit.Features/Test/Lifecycle/Targets/Actions/InitLifecycleTarget.cs

@@ -55,6 +55,8 @@ runner with the `--use-runner` argument.
 (Hint: If you'd like to regenerate your files, you can use the `--overwrite-existing` option to overwrite the files that are in place now.)";
 
 
+            Log<InitLifecycleTarget>.G().LogInformation(message);
+
 
         }
     }

src/CodeQLToolkit.Features/Validation/Commands/Targets/CheckQueriesCommandTarget.cs

@@ -0,0 +1,81 @@
+using CodeQLToolkit.Features.Query.Commands.Targets;
+using CodeQLToolkit.Features.Validation.Models;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodeQLToolkit.Features.Validation.Commands.Targets
+{
+    public class CheckQueriesCommandTarget : CommandTarget
+    {
+
+        public bool PrettyPrint { get; set; }
+
+
+        public override void Run()
+        {
+            Log<CheckQueriesCommandTarget>.G().LogInformation($"Validating query metadata for {Language}...");
+
+            using (Process process = new Process())
+            {
+                process.StartInfo.FileName = "codeql";
+                process.StartInfo.UseShellExecute = false;
+                process.StartInfo.WorkingDirectory = Base;
+                process.StartInfo.RedirectStandardOutput = true;
+                process.StartInfo.RedirectStandardError = false;
+                process.StartInfo.Arguments = $"query compile --format json  -n {Language}";
+                process.Start();
+
+                var output = process.StandardOutput.ReadToEnd();
+
+                process.WaitForExit();
+
+                if (process.ExitCode != 0)
+                {
+                    DieWithError($"Fatal error. Please check error output.");
+                }
+
+                var results = JsonConvert.DeserializeObject<List<CodeQLCliResponseModel>>(output);
+
+                bool shouldFail = false;
+
+                if (results != null)
+                {
+                    foreach (var r in results)
+                    {
+                        foreach (var message in r.messages)
+                        {
+                            if (message.severity == "WARNING" || message.severity == "ERROR")
+                            {
+                                shouldFail = true;
+
+                                if (PrettyPrint)
+                                {
+                                    Console.WriteLine($"❌ [{message.severity}] {message.message}: {message.position.fileName}:{message.position.line},{message.position.column}-{message.position.endColumn},{message.position.endColumn}");
+                                }
+                                else
+                                {
+                                    Log<CheckQueriesCommandTarget>.G().LogWarning($"[{message.severity}] {message.message}: {message.position.fileName}:{message.position.line},{message.position.column}-{message.position.endColumn},{message.position.endColumn}");
+                                }
+
+                            }
+                        }
+                    }
+                }
+
+                if(shouldFail && !PrettyPrint )
+                {
+                    DieWithError("One or more validation errors found.");
+                }
+
+
+
+            }
+
+        }
+    }
+}

src/CodeQLToolkit.Features/Validation/Commands/ValidationCommandFeature.cs

@@ -1,4 +1,5 @@
-using CodeQLToolkit.Shared.Utils;
+using CodeQLToolkit.Features.Validation.Commands.Targets;
+using CodeQLToolkit.Shared.Utils;
 using System.CommandLine;
 
 namespace CodeQLToolkit.Features.Test.Commands
@@ -29,27 +30,29 @@ public void Register(Command parentCommand)
             var runCommand = new Command("run", "Functions pertaining running validation commands.");
             parentCommand.Add(runCommand);
 
-            var getMatrixTestCommand = new Command("check-metadsata", "Checks the query metadata for the specified queries.");
+            var checkQueryQueriesCommand = new Command("check-queries", "Checks the query metadata for the specified language.");
 
             var languageOption = new Option<string>("--language", $"The language to run tests for.") { IsRequired = true }.FromAmong(SupportedLangauges.Select(x => x.ToOptionString()).ToArray());
-            var matrixOSVersion = new Option<string>("--os-version", () => "ubuntu-latest", "A comma-seperated list of operating systems to use. Example: `ubuntu-latest`.") { IsRequired = true };
-            getMatrixTestCommand.Add(matrixOSVersion);
-
-            runCommand.Add(getMatrixTestCommand);
+            var prettyPrintOption = new Option<bool>("--pretty-print", () => false, "Pretty prints error output in a pretty compact format.") { IsRequired = true };
 
+            checkQueryQueriesCommand.Add(languageOption);
+            checkQueryQueriesCommand.Add(prettyPrintOption);
 
-            getMatrixTestCommand.SetHandler(() =>
-            {
-                Log<ValidationCommandFeature>.G().LogInformation("Executing validate-unit-tests command...");
-
-                //new ValidateUnitTestsCommand()
-                //{
-                //    ResultsDirectory = resultsDirectory,
-                //    PrettyPrint = prettyPrint
-                //}.Run();
+            runCommand.Add(checkQueryQueriesCommand);
 
 
-            });
+            checkQueryQueriesCommand.SetHandler((language, basePath, prettyPrint) =>
+            {
+                Log<ValidationCommandFeature>.G().LogInformation("Executing check-query-metadata command...");
+
+                new CheckQueriesCommandTarget()
+                {
+                    Base = basePath,
+                    Language = language,
+                    PrettyPrint = prettyPrint,                    
+                }.Run();
+
+            }, languageOption, Globals.BasePathOption, prettyPrintOption);
         }
 
         public int Run()

src/CodeQLToolkit.Features/Validation/Lifecycle/BaseLifecycleTarget.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodeQLToolkit.Features.Validation.Lifecycle
+{
+    abstract internal class BaseLifecycleTarget : ILifecycleTarget
+    {
+        public string UseRunner { get; set; }
+
+    }
+}

src/CodeQLToolkit.Features/Validation/Lifecycle/Targets/Actions/InitLifecycleTarget.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodeQLToolkit.Features.Validation.Lifecycle.Targets.Actions
+{
+    [AutomationType(AutomationType.ACTIONS)]
+    internal class InitLifecycleTarget : BaseLifecycleTarget
+    {
+        public InitLifecycleTarget()
+        {
+            AutomationType = AutomationType.ACTIONS;
+        }
+
+        public override void Run()
+        {
+            Log<InitLifecycleTarget>.G().LogInformation("Running init command...");
+
+            // temporarily disable the language resolution 
+            var tmpLanguage = Language;
+            Language = null;
+
+            WriteTemplateIfOverwriteOrNotExists("validate-query-metadata", Path.Combine(Base, ".github", "workflows", $"validate-codeql-queries-{tmpLanguage}.yml"), $"Validate CodeQL Queries ({Language})", new
+            {
+                useRunner = UseRunner,
+                language = tmpLanguage,
+            });
+
+            Language = tmpLanguage;
+
+            var message = @"------------------------------------------
+Your repository now has CodeQL Query Validation installed in `.github/workflows/`. Please ensure to initialize CodeQL 
+testing before using this workflow with `qlt test init`. 
+
+Note that, by default, your runner the `ubuntu-latest` runner. 
+
+You can modify default runner by adjusting the `--use-runner` argument. 
+
+In addition to using QLT to generate your files you can also directly edit this file to fine tune its settings. 
+            
+(Hint: If you'd like to regenerate your files, you can use the `--overwrite-existing` option to overwrite the files that are in place now.)";
+
+            Log<InitLifecycleTarget>.G().LogInformation(message);
+        }
+    }
+}

src/CodeQLToolkit.Features/Validation/Lifecycle/ValidationLifecycleFeature.cs

@@ -0,0 +1,75 @@
+using CodeQLToolkit.Features.Test.Lifecycle;
+using CodeQLToolkit.Shared.Utils;
+using System;
+using System.Collections.Generic;
+using System.CommandLine;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodeQLToolkit.Features.Validation.Lifecycle
+{
+    internal class ValidationLifecycleFeature : FeatureBase, IToolkitLifecycleFeature
+    {
+        public ValidationLifecycleFeature()
+        {
+            FeatureName = "Validation";
+        }
+
+        public override LanguageType[] SupportedLangauges
+        {
+            get => new LanguageType[] {
+            LanguageType.C,
+            LanguageType.CPP,
+            LanguageType.CSHARP,
+            LanguageType.JAVA,
+            LanguageType.JAVASCRIPT,
+            LanguageType.GO,
+            LanguageType.RUBY,
+            LanguageType.PYTHON
+        };
+        }
+
+        public void Register(Command parentCommand)
+        {
+            Log<ValidationLifecycleFeature>.G().LogInformation("Registering lifecycle submodule.");
+
+            var initCommand = new Command("init", "Initialize validation checking in this repository.");
+
+            var overwriteExistingOption = new Option<bool>("--overwrite-existing", () => false, "Overwrite exiting files (if they exist).");
+            var languageOption = new Option<string>("--language", $"The language to generate automation for.") { IsRequired = true }.FromAmong(SupportedLangauges.Select(x => x.ToOptionString()).ToArray());
+            var useRunnerOption = new Option<string>("--use-runner", () => "ubuntu-latest", "The runner(s) to use. Should be a comma-seperated list of actions runners.");
+
+            initCommand.AddOption(overwriteExistingOption);
+            initCommand.AddOption(languageOption);
+            initCommand.AddOption(useRunnerOption);
+
+            parentCommand.Add(initCommand);
+
+            initCommand.SetHandler((basePath, automationType, overwriteExisting, language, useRunner) =>
+            {
+                Log<TestLifecycleFeature>.G().LogInformation("Executing init command...");
+
+                //
+                // dispatch at runtime to the correct automation type
+                //
+                var featureTarget = AutomationFeatureFinder.FindTargetForAutomationType<BaseLifecycleTarget>(AutomationTypeHelper.AutomationTypeFromString(automationType));
+
+                // setup common params 
+                featureTarget.FeatureName = FeatureName;
+                featureTarget.Base = basePath;
+                featureTarget.UseRunner = useRunner;
+                featureTarget.OverwriteExisting = overwriteExisting;
+                featureTarget.Language = language;
+                featureTarget.Run();
+
+            }, Globals.BasePathOption, Globals.AutomationTypeOption, overwriteExistingOption, languageOption, useRunnerOption);
+
+        }
+
+        public int Run()
+        {
+            throw new NotImplementedException();
+        }
+    }
+}