From 8a8b8da6c456ded3fec6079cd7b2f46d763ecc8e Mon Sep 17 00:00:00 2001 From: Prangshuman Das <81367237+t-prda@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:56:35 +0200 Subject: [PATCH] Show token consumption in AI Test Toolkit (#2022) #### Summary Log token consumption while running the tests using AI Test Toolkit Screenshots: ![image](https://github.com/user-attachments/assets/8bea52a9-20aa-47c1-ace7-f9a52ebfd996) ![image](https://github.com/user-attachments/assets/3c84178d-b480-4738-b0d5-4ae1a9281552) ![image](https://github.com/user-attachments/assets/26175d46-c6e7-4f17-88b7-66912a5d4b21) #### Work Item(s) Fixes [AB#540454](https://dynamicssmb2.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_workitems/edit/540454) --- .../AI/src/Azure OpenAI/AOAIToken.Codeunit.al | 12 +++++++++++ .../Azure OpenAI/AzureOpenAIImpl.Codeunit.al | 5 +++++ .../src/AITTestRunIteration.Codeunit.al | 19 +++++++++++++++++ .../src/API/AITLogEntryAPI.Page.al | 4 ++++ .../src/Logs/AITLogEntries.Page.al | 3 +++ .../src/Logs/AITLogEntry.Table.al | 5 +++++ .../Logs/AITTestMethodLinesCompare.Page.al | 13 ++++++++++++ .../src/Logs/AITTestSuiteCompare.Page.al | 13 ++++++++++++ .../src/TestSuite/AITTestMethodLine.Table.al | 16 ++++++++++++++ .../src/TestSuite/AITTestMethodLines.Page.al | 3 +++ .../src/TestSuite/AITTestSuite.Page.al | 21 ++++++++++++++++--- .../src/TestSuite/AITTestSuite.Table.al | 16 ++++++++++++++ .../src/TestSuite/AITTestSuiteMgt.Codeunit.al | 1 + 13 files changed, 128 insertions(+), 3 deletions(-) diff --git a/src/System Application/App/AI/src/Azure OpenAI/AOAIToken.Codeunit.al b/src/System Application/App/AI/src/Azure OpenAI/AOAIToken.Codeunit.al index 9de2722196..9367531908 100644 --- a/src/System Application/App/AI/src/Azure OpenAI/AOAIToken.Codeunit.al +++ b/src/System Application/App/AI/src/Azure OpenAI/AOAIToken.Codeunit.al @@ -53,4 +53,16 @@ codeunit 7759 "AOAI Token" begin exit(AzureOpenAIImpl.GetTokenCount(Input, Encodingp50kbaseLbl)); end; + + + /// + /// Gets the total tokens used till now for the server session. + /// The total tokens used is aggregated for all the models. + /// Note: this method is expected to change in future. + /// + /// The total token consumed for the session. + procedure GetTotalServerSessionTokensConsumed(): Integer + begin + exit(AzureOpenAIImpl.GetTotalServerSessionTokensConsumed()); + end; } \ No newline at end of file diff --git a/src/System Application/App/AI/src/Azure OpenAI/AzureOpenAIImpl.Codeunit.al b/src/System Application/App/AI/src/Azure OpenAI/AzureOpenAIImpl.Codeunit.al index 72e65015e6..eb6a83aade 100644 --- a/src/System Application/App/AI/src/Azure OpenAI/AzureOpenAIImpl.Codeunit.al +++ b/src/System Application/App/AI/src/Azure OpenAI/AzureOpenAIImpl.Codeunit.al @@ -705,6 +705,11 @@ codeunit 7772 "Azure OpenAI Impl" TokenCount := ALCopilotFunctions.GptTokenCount(Input, Encoding); end; + procedure GetTotalServerSessionTokensConsumed(): Integer + begin + exit(SessionInformation.AITokensUsed); + end; + [NonDebuggable] internal procedure IsTenantAllowlistedForFirstPartyCopilotCalls(): Boolean var diff --git a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al index 0e8ef72a84..4565bd168a 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al @@ -5,6 +5,7 @@ namespace System.TestTools.AITestToolkit; +using System.AI; using System.TestTools.TestRunner; codeunit 149042 "AIT Test Run Iteration" @@ -19,6 +20,8 @@ codeunit 149042 "AIT Test Run Iteration" ActiveAITTestSuite: Record "AIT Test Suite"; GlobalTestMethodLine: Record "Test Method Line"; NoOfInsertedLogEntries: Integer; + GlobalAITokenUsedByLastTestMethodLine: Integer; + GlobalSessionAITokenUsed: Integer; trigger OnRun() begin @@ -27,6 +30,7 @@ codeunit 149042 "AIT Test Run Iteration" SetAITTestMethodLine(Rec); NoOfInsertedLogEntries := 0; + GlobalAITokenUsedByLastTestMethodLine := 0; InitializeAITTestMethodLineForRun(Rec, ActiveAITTestSuite); SetAITTestSuite(ActiveAITTestSuite); @@ -115,6 +119,11 @@ codeunit 149042 "AIT Test Run Iteration" exit(GlobalTestMethodLine); end; + procedure GetAITokenUsedByLastTestMethodLine(): Integer + begin + exit(GlobalAITokenUsedByLastTestMethodLine); + end; + [InternalEvent(false)] procedure OnBeforeRunIteration(var AITTestSuite: Record "AIT Test Suite"; var AITTestMethodLine: Record "AIT Test Method Line") begin @@ -124,6 +133,7 @@ codeunit 149042 "AIT Test Run Iteration" local procedure OnBeforeTestMethodRun(var CurrentTestMethodLine: Record "Test Method Line"; CodeunitID: Integer; CodeunitName: Text[30]; FunctionName: Text[128]; FunctionTestPermissions: TestPermissions) var AITContextCU: Codeunit "AIT Test Context Impl."; + AOAIToken: Codeunit "AOAI Token"; begin if ActiveAITTestSuite.Code = '' then exit; @@ -132,6 +142,10 @@ codeunit 149042 "AIT Test Run Iteration" GlobalTestMethodLine := CurrentTestMethodLine; + // Update AI Token Consumption + GlobalAITokenUsedByLastTestMethodLine := 0; + GlobalSessionAITokenUsed := AOAIToken.GetTotalServerSessionTokensConsumed(); + AITContextCU.StartRunProcedureScenario(); end; @@ -139,6 +153,7 @@ codeunit 149042 "AIT Test Run Iteration" local procedure OnAfterTestMethodRun(var CurrentTestMethodLine: Record "Test Method Line"; CodeunitID: Integer; CodeunitName: Text[30]; FunctionName: Text[128]; FunctionTestPermissions: TestPermissions; IsSuccess: Boolean) var AITContextCU: Codeunit "AIT Test Context Impl."; + AOAIToken: Codeunit "AOAI Token"; begin if ActiveAITTestSuite.Code = '' then exit; @@ -147,6 +162,10 @@ codeunit 149042 "AIT Test Run Iteration" exit; GlobalTestMethodLine := CurrentTestMethodLine; + + // Update AI Token Consumption + GlobalAITokenUsedByLastTestMethodLine := AOAIToken.GetTotalServerSessionTokensConsumed() - GlobalSessionAITokenUsed; + AITContextCU.EndRunProcedureScenario(CurrentTestMethodLine, IsSuccess); Commit(); end; diff --git a/src/Tools/AI Test Toolkit/src/API/AITLogEntryAPI.Page.al b/src/Tools/AI Test Toolkit/src/API/AITLogEntryAPI.Page.al index 1a0cc3d7c7..d21a0d8f52 100644 --- a/src/Tools/AI Test Toolkit/src/API/AITLogEntryAPI.Page.al +++ b/src/Tools/AI Test Toolkit/src/API/AITLogEntryAPI.Page.al @@ -50,6 +50,10 @@ page 149038 "AIT Log Entry API" { Caption = 'Version'; } + field(tokensConsumed; Rec."Tokens Consumed") + { + Caption = 'Total Tokens Consumed'; + } field("startTime"; Rec."Start Time") { Caption = 'Start Time'; diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al index 3a5108e963..5baf975084 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al @@ -101,6 +101,9 @@ page 149033 "AIT Log Entries" Message(Rec.GetOutputBlob()); end; } + field("Tokens Consumed"; Rec."Tokens Consumed") + { + } field(TestRunDuration; TestRunDuration) { Caption = 'Duration'; diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al index b4fe826903..2cdb58d12b 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al @@ -158,6 +158,11 @@ table 149034 "AIT Log Entry" { Caption = 'Output Data'; } + field(50; "Tokens Consumed"; Integer) + { + Caption = 'Total Tokens Consumed'; + ToolTip = 'Specifies the aggregated number of tokens consumed by the test. This is applicable only when using Microsoft AI Module.'; + } } keys diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITTestMethodLinesCompare.Page.al b/src/Tools/AI Test Toolkit/src/Logs/AITTestMethodLinesCompare.Page.al index 7a5f898620..696eee149a 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITTestMethodLinesCompare.Page.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITTestMethodLinesCompare.Page.al @@ -74,6 +74,11 @@ page 149035 "AIT Test Method Lines Compare" Caption = 'Total Duration (ms)'; ToolTip = 'Specifies Total Duration of the test for given version.'; } + label(TokensConsumed) + { + Caption = 'Total Tokens Consumed'; + ToolTip = 'Specifies the aggregated number of tokens consumed by the test. This is applicable only when using Microsoft AI Module.'; + } } group("Latest Version") { @@ -107,6 +112,10 @@ page 149035 "AIT Test Method Lines Compare" ToolTip = 'Specifies Total Duration of the tests for this version.'; ShowCaption = false; } + field("Tokens Consumed"; Rec."Tokens Consumed") + { + ShowCaption = false; + } } group("Base Version") { @@ -142,6 +151,10 @@ page 149035 "AIT Test Method Lines Compare" Caption = 'Total Duration Base (ms)'; ShowCaption = false; } + field("Tokens Consumed - Base"; Rec."Tokens Consumed - Base") + { + ShowCaption = false; + } } } } diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITTestSuiteCompare.Page.al b/src/Tools/AI Test Toolkit/src/Logs/AITTestSuiteCompare.Page.al index 5fa587d94c..94322a0460 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITTestSuiteCompare.Page.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITTestSuiteCompare.Page.al @@ -75,6 +75,11 @@ page 149036 "AIT Test Suite Compare" Caption = 'Total Duration (ms)'; ToolTip = 'Specifies Total Duration of the tests for the version.'; } + label(TokensConsumed) + { + Caption = 'Total Tokens Consumed'; + ToolTip = 'Specifies the aggregated number of tokens consumed by the test. This is applicable only when using Microsoft AI Module.'; + } } group("Latest Version") { @@ -110,6 +115,10 @@ page 149036 "AIT Test Suite Compare" ToolTip = 'Specifies Total Duration of the tests for this version.'; ShowCaption = false; } + field("Tokens Consumed"; Rec."Tokens Consumed") + { + ShowCaption = false; + } } group("Base Version") { @@ -146,6 +155,10 @@ page 149036 "AIT Test Suite Compare" Caption = 'Total Duration Base (ms)'; ShowCaption = false; } + field("Tokens Consumed - Base"; Rec."Tokens Consumed - Base") + { + ShowCaption = false; + } } } } diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al index 72142d7a8b..da4ea0598d 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al @@ -171,6 +171,22 @@ table 149032 "AIT Test Method Line" Caption = 'AL Test Suite'; Editable = false; } + field(120; "Tokens Consumed"; Integer) + { + Caption = 'Total Tokens Consumed'; + ToolTip = 'Specifies the number of tokens consumed by the test in the current version. This is applicable only when using Microsoft AI Module.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + } + field(121; "Tokens Consumed - Base"; Integer) + { + Caption = 'Tokens Consumed - Base'; + ToolTip = 'Specifies the number of tokens consumed by the test in the base version. This is applicable only when using Microsoft AI Module.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Base Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + } } keys diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al index d94d12805a..02c7a1d261 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al @@ -81,6 +81,9 @@ page 149034 "AIT Test Method Lines" field(Duration; Rec."Total Duration (ms)") { } + field("Tokens Consumed"; Rec."Tokens Consumed") + { + } field(AvgDuration; AITTestSuiteMgt.GetAvgDuration(Rec)) { Caption = 'Average Duration (ms)'; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al index 6542dbf952..bd3b4462d1 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al @@ -126,6 +126,15 @@ page 149031 "AIT Test Suite" Caption = 'Average Duration'; ToolTip = 'Specifies the average time taken by the tests in the test suite.'; } + field("Tokens Consumed"; Rec."Tokens Consumed") + { + } + field("Average Tokens Consumed"; AvgTokensConsumed) + { + Editable = false; + Caption = 'Average Tokens Consumed'; + ToolTip = 'Specifies the average number of tokens consumed by the tests in the last run.'; + } } } @@ -260,6 +269,7 @@ page 149031 "AIT Test Suite" AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; EnableActions: Boolean; AvgTimeDuration: Duration; + AvgTokensConsumed: Integer; TotalDuration: Duration; PageCaptionLbl: Label 'AI Test'; TestRunnerDisplayName: Text; @@ -284,7 +294,7 @@ page 149031 "AIT Test Suite" TestSuiteMgt: Codeunit "Test Suite Mgt."; begin UpdateTotalDuration(); - UpdateAverageExecutionTime(); + UpdateAverages(); TestRunnerDisplayName := TestSuiteMgt.GetTestRunnerDisplayName(Rec."Test Runner Id"); end; @@ -294,12 +304,17 @@ page 149031 "AIT Test Suite" TotalDuration := Rec."Total Duration (ms)"; end; - local procedure UpdateAverageExecutionTime() + local procedure UpdateAverages() begin - Rec.CalcFields("No. of Tests Executed", "Total Duration (ms)", "No. of Tests Executed - Base", "Total Duration (ms) - Base"); + Rec.CalcFields("No. of Tests Executed", "Total Duration (ms)", "Tokens Consumed"); if Rec."No. of Tests Executed" > 0 then AvgTimeDuration := Rec."Total Duration (ms)" div Rec."No. of Tests Executed" else AvgTimeDuration := 0; + + if Rec."No. of Tests Executed" > 0 then + AvgTokensConsumed := Rec."Tokens Consumed" div Rec."No. of Tests Executed" + else + AvgTokensConsumed := 0; end; } \ No newline at end of file diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al index 786a23b2d1..da05050dcc 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al @@ -151,6 +151,14 @@ table 149030 "AIT Test Suite" FieldClass = FlowField; CalcFormula = count("AIT Log Entry" where("Test Suite Code" = field("Code"), "Version" = field("Version"))); } + field(24; "Tokens Consumed"; Integer) + { + Caption = 'Total Tokens Consumed'; + ToolTip = 'Specifies the aggregated number of tokens consumed by the test in the current version. This is applicable only when using Microsoft AI Module.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Code"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + } field(31; "No. of Tests Executed - Base"; Integer) { Caption = 'No. of Tests Executed'; @@ -183,6 +191,14 @@ table 149030 "AIT Test Suite" FieldClass = FlowField; CalcFormula = sum("AIT Log Entry"."Duration (ms)" where("Test Suite Code" = field("Code"), Version = field("Base Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); } + field(35; "Tokens Consumed - Base"; Integer) + { + Caption = 'Total Tokens Consumed - Base'; + ToolTip = 'Specifies the aggregated number of tokens consumed by the test in the base version. This is applicable only when using Microsoft AI Module.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Code"), Version = field("Base Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + } field(50; "Test Runner Id"; Integer) { Caption = 'Test Runner Id'; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index 59fbd1a3c2..2fa30d2af4 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -339,6 +339,7 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry.SetOutputBlob(TestOutput); AITLogEntry."Procedure Name" := CurrentTestMethodLine.Function; + AITLogEntry."Tokens Consumed" := AITTestRunIteration.GetAITokenUsedByLastTestMethodLine(); AITLogEntry.Insert(true); Commit();