[AI Test Tool] Add Multi-Turn Accuracy & Export

src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al

@@ -123,6 +123,15 @@ codeunit 149044 "AIT Test Context"
         AITTestContextImpl.SetTestMetric(TestMetric);
     end;
 
+    /// <summary>
+    /// Sets the accuracy of the test.
+    /// </summary>
+    /// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
+    procedure SetAccuracy(Accuracy: Decimal)
+    begin
+        AITTestContextImpl.SetAccuracy(Accuracy);
+    end;
+
     var
         AITTestContextImpl: Codeunit "AIT Test Context Impl.";
 }

src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al

@@ -18,9 +18,12 @@ codeunit 149043 "AIT Test Context Impl."
     var
         AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
         GlobalTestOutputJson: Codeunit "Test Output Json";
+        Accuracy: Decimal;
         CurrentTurn: Integer;
         NumberOfTurns: Integer;
         IsMultiTurn: Boolean;
+        AccuracySetManually: Boolean;
+        AccuracyErr: Label 'Accuracy must be between 0.0 and 1.0.';
         AnswerTok: Label 'answer', Locked = true;
         ContextTok: Label 'context', Locked = true;
         GroundTruthTok: Label 'ground_truth', Locked = true;
@@ -146,6 +149,33 @@ codeunit 149043 "AIT Test Context Impl."
         SetSuiteTestOutput(CurrentTestOutputJson.ToText());
     end;
 
+    /// <summary>
+    /// Sets the accuracy of the test.
+    /// </summary>
+    /// <param name="AccuracyPct">The accuracy as a decimal between 0 and 1.</param>
+    procedure SetAccuracy(AccuracyPct: Decimal)
+    begin
+        if (AccuracyPct < 0) or (AccuracyPct > 1.0) then
+            Error(AccuracyErr);
+
+        AccuracySetManually := true;
+        Accuracy := AccuracyPct;
+    end;
+
+    /// <summary>
+    /// Gets the accuracy of the test.
+    /// </summary>
+    /// <returns>True if the accuracy was set, false otherwise.</returns>
+    procedure GetAccuracy(var AccuracyPct: Decimal): Boolean
+    begin
+        if AccuracySetManually then begin
+            AccuracyPct := Accuracy;
+            exit(true);
+        end;
+
+        exit(false);
+    end;
+
     /// <summary>
     /// Sets to next turn for multiturn testing.
     /// </summary>
@@ -155,7 +185,7 @@ codeunit 149043 "AIT Test Context Impl."
         if not IsMultiTurn then
             exit(false);
 
-        if CurrentTurn + 1 > NumberOfTurns then
+        if CurrentTurn > NumberOfTurns then
             exit(false);
 
         CurrentTurn := CurrentTurn + 1;
@@ -164,14 +194,23 @@ codeunit 149043 "AIT Test Context Impl."
     end;
 
     /// <summary>
-    /// Gets the current turn for multiturn testing. Turns start from turn 0.
+    /// Gets the current turn for multiturn testing. Turns start from turn 1.
     /// </summary>
     /// <returns>The current turn number.</returns>
     procedure GetCurrentTurn(): Integer
     begin
         exit(CurrentTurn);
     end;
 
+    /// <summary>
+    /// Gets the total number of turns for multiturn testing.
+    /// </summary>
+    /// <returns>The total number of turns for the line.</returns>
+    procedure GetNumberOfTurns(): Integer
+    begin
+        exit(NumberOfTurns);
+    end;
+
     /// <summary>
     /// This method starts the scope of the Run Procedure scenario.
     /// </summary>
@@ -205,12 +244,16 @@ codeunit 149043 "AIT Test Context Impl."
         TestInput: Codeunit "Test Input";
         TurnsInputJson: Codeunit "Test Input Json";
     begin
-        CurrentTurn := 0;
+        AccuracySetManually := false;
+        Accuracy := 0;
+        CurrentTurn := 1;
         GlobalTestOutputJson.Initialize();
         TurnsInputJson := TestInput.GetTestInput().ElementExists(TurnsTok, IsMultiTurn);
 
         if IsMultiTurn then
-            NumberOfTurns := TurnsInputJson.GetElementCount() - 1;
+            NumberOfTurns := TurnsInputJson.GetElementCount()
+        else
+            NumberOfTurns := 1;
     end;
 
     /// <summary>
@@ -223,7 +266,7 @@ codeunit 149043 "AIT Test Context Impl."
         TestInput: Codeunit "Test Input";
     begin
         if IsMultiTurn then
-            TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn).Element(ElementName)
+            TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn - 1).Element(ElementName)
         else
             TestInputJson := TestInput.GetTestInput(ElementName);
     end;

src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al

@@ -21,6 +21,9 @@ codeunit 149042 "AIT Test Run Iteration"
         GlobalTestMethodLine: Record "Test Method Line";
         NoOfInsertedLogEntries: Integer;
         GlobalAITokenUsedByLastTestMethodLine: Integer;
+        GlobalNumberOfTurnsForLastTestMethodLine: Integer;
+        GlobalNumberOfTurnsPassedForLastTestMethodLine: Integer;
+        GlobalTestAccuracy: Decimal;
         GlobalSessionAITokenUsed: Integer;
 
     trigger OnRun()
@@ -124,6 +127,21 @@ codeunit 149042 "AIT Test Run Iteration"
         exit(GlobalAITokenUsedByLastTestMethodLine);
     end;
 
+    procedure GetNumberOfTurnsForLastTestMethodLine(): Integer
+    begin
+        exit(GlobalNumberOfTurnsForLastTestMethodLine);
+    end;
+
+    procedure GetNumberOfTurnsPassedForLastTestMethodLine(): Integer
+    begin
+        exit(GlobalNumberOfTurnsPassedForLastTestMethodLine);
+    end;
+
+    procedure GetAccuracyForLastTestMethodLine(): Decimal
+    begin
+        exit(GlobalTestAccuracy);
+    end;
+
     [InternalEvent(false)]
     procedure OnBeforeRunIteration(var AITTestSuite: Record "AIT Test Suite"; var AITTestMethodLine: Record "AIT Test Method Line")
     begin
@@ -144,6 +162,14 @@ codeunit 149042 "AIT Test Run Iteration"
 
         // Update AI Token Consumption
         GlobalAITokenUsedByLastTestMethodLine := 0;
+
+        // Update Turns
+        GlobalNumberOfTurnsPassedForLastTestMethodLine := 0;
+        GlobalNumberOfTurnsForLastTestMethodLine := 1;
+
+        // Update Test Accuracy
+        GlobalTestAccuracy := 0.0;
+
         GlobalSessionAITokenUsed := AOAIToken.GetTotalServerSessionTokensConsumed();
 
         AITContextCU.StartRunProcedureScenario();
@@ -154,6 +180,7 @@ codeunit 149042 "AIT Test Run Iteration"
     var
         AITContextCU: Codeunit "AIT Test Context Impl.";
         AOAIToken: Codeunit "AOAI Token";
+        Accuracy: Decimal;
     begin
         if ActiveAITTestSuite.Code = '' then
             exit;
@@ -166,6 +193,19 @@ codeunit 149042 "AIT Test Run Iteration"
         // Update AI Token Consumption
         GlobalAITokenUsedByLastTestMethodLine := AOAIToken.GetTotalServerSessionTokensConsumed() - GlobalSessionAITokenUsed;
 
+        // Update Turns
+        GlobalNumberOfTurnsForLastTestMethodLine := AITContextCU.GetNumberOfTurns();
+        GlobalNumberOfTurnsPassedForLastTestMethodLine := AITContextCU.GetCurrentTurn();
+
+        if not IsSuccess then
+            GlobalNumberOfTurnsPassedForLastTestMethodLine -= 1;
+
+        // Update Test Accuracy
+        if AITContextCU.GetAccuracy(Accuracy) then
+            GlobalTestAccuracy := Accuracy
+        else
+            GlobalTestAccuracy := GlobalNumberOfTurnsPassedForLastTestMethodLine / GlobalNumberOfTurnsForLastTestMethodLine;
+
         AITContextCU.EndRunProcedureScenario(CurrentTestMethodLine, IsSuccess);
         Commit();
     end;

src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al

@@ -68,6 +68,23 @@ page 149033 "AIT Log Entries"
                 {
                     StyleExpr = StatusStyleExpr;
                 }
+                field(Accuracy; Rec.Accuracy)
+                {
+                }
+                field("No. of Turns Passed"; Rec."No. of Turns Passed")
+                {
+                    Visible = false;
+                }
+                field("No. of Turns Executed"; Rec."No. of Turns Executed")
+                {
+                    Visible = false;
+                }
+                field(TurnsText; TurnsText)
+                {
+                    StyleExpr = TurnsStyleExpr;
+                    Caption = 'No. of Turns Passed';
+                    ToolTip = 'Specifies the number of turns that passed out of the total number of turns.';
+                }
                 field("Orig. Status"; Rec."Original Status")
                 {
                     Visible = false;
@@ -253,6 +270,19 @@ page 149033 "AIT Log Entries"
                     Page.Run(Page::"AIT Test Data Compare", Rec);
                 end;
             }
+            action("Export Results")
+            {
+                Caption = 'Export Results';
+                Image = Export;
+                ToolTip = 'Exports the results.';
+
+                trigger OnAction()
+                var
+                    AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
+                begin
+                    AITTestSuiteMgt.ExportResults(Rec);
+                end;
+            }
         }
         area(Promoted)
         {
@@ -279,6 +309,9 @@ page 149033 "AIT Log Entries"
                 actionref("View Test Data_Promoted"; "View Test Data")
                 {
                 }
+                actionref("Export Results_Promoted"; "Export Results")
+                {
+                }
             }
         }
     }
@@ -287,20 +320,26 @@ page 149033 "AIT Log Entries"
         ClickToShowLbl: Label 'Show data input';
         DoYouWantToDeleteQst: Label 'Do you want to delete all entries within the filter?';
         InputText: Text;
+        TurnsText: Text;
         OutputText: Text;
         ErrorMessage: Text;
         ErrorCallStack: Text;
         StatusStyleExpr: Text;
+        TurnsStyleExpr: Text;
         TestRunDuration: Duration;
         IsFilteredToErrors: Boolean;
         ShowSensitiveData: Boolean;
 
     trigger OnAfterGetRecord()
+    var
+        AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
     begin
         TestRunDuration := Rec."Duration (ms)";
+        TurnsText := AITTestSuiteMgt.GetTurnsAsText(Rec);
         SetInputOutputDataFields();
         SetErrorFields();
         SetStatusStyleExpr();
+        SetTurnsStyleExpr();
     end;
 
     local procedure SetStatusStyleExpr()
@@ -315,6 +354,18 @@ page 149033 "AIT Log Entries"
         end;
     end;
 
+    local procedure SetTurnsStyleExpr()
+    begin
+        case Rec."No. of Turns Passed" of
+            Rec."No. of Turns Executed":
+                TurnsStyleExpr := 'Favorable';
+            0:
+                TurnsStyleExpr := 'Unfavorable';
+            else
+                TurnsStyleExpr := 'Ambiguous';
+        end;
+    end;
+
     local procedure SetErrorFields()
     begin
         ErrorMessage := '';

src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al

@@ -159,6 +159,21 @@ table 149034 "AIT Log Entry"
         {
             Caption = 'Output Data';
         }
+        field(40; "No. of Turns Executed"; Integer)
+        {
+            Caption = 'Total number of turns';
+            ToolTip = 'Specifies the total number of turns executed.';
+        }
+        field(41; "No. of Turns Passed"; Integer)
+        {
+            Caption = 'Number of turns passed';
+            ToolTip = 'Specifies the number of turns passed.';
+        }
+        field(45; Accuracy; Decimal)
+        {
+            Caption = 'Accuracy';
+            ToolTip = 'Specifies the accuracy of the test line.';
+        }
         field(50; "Tokens Consumed"; Integer)
         {
             Caption = 'Total Tokens Consumed';

src/Tools/AI Test Toolkit/src/Logs/AITResults.Report.al

@@ -0,0 +1,73 @@
+// ------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+// ------------------------------------------------------------------------------------------------
+
+namespace System.TestTools.AITestToolkit;
+
+report 149030 "AIT Results"
+{
+    Caption = 'AI Test Results';
+    ApplicationArea = All;
+    UsageCategory = Tasks;
+    DefaultLayout = Excel;
+    ExcelLayout = 'Results.xlsx';
+
+    dataset
+    {
+        dataitem(Results; "AIT Log Entry")
+        {
+
+            RequestFilterFields = Version;
+            RequestFilterHeading = 'AI Test Log Entries';
+
+            column(CodeunitID; Results."Codeunit ID")
+            {
+            }
+            column(Name; Results."Codeunit Name")
+            {
+            }
+            column(TestName; Results."Procedure Name")
+            {
+            }
+            column(Status; Results.Status)
+            {
+            }
+            column(Accuracy; Results.Accuracy)
+            {
+            }
+            column(TurnsExecuted; Results."No. of Turns Executed")
+            {
+            }
+            column(TurnsPassed; Results."No. of Turns Passed")
+            {
+            }
+            column(Input; Input)
+            {
+            }
+            column(Output; Output)
+            {
+            }
+            column(Error_Message; ErrorMessage)
+            {
+            }
+            column(Error; ErrorCallstack)
+            {
+            }
+
+            trigger OnAfterGetRecord()
+            begin
+                Input := Results.GetInputBlob();
+                Output := Results.GetOutputBlob();
+                ErrorMessage := Results.GetMessage();
+                ErrorCallstack := Results.GetErrorCallStack();
+            end;
+        }
+    }
+
+    var
+        Input: Text;
+        Output: Text;
+        ErrorMessage: Text;
+        ErrorCallstack: Text;
+}