Add telemetry for test results

Actions/AnalyzeTests/TestResultAnalyzer.ps1

@@ -9,6 +9,7 @@ $statusSkipped = " :question:"
 # Returns both a summary part and a failures part
 $mdHelperPath = Join-Path -Path $PSScriptRoot -ChildPath "..\MarkDownHelper.psm1"
 Import-Module $mdHelperPath
+Import-Module (Join-Path $PSScriptRoot '..\TelemetryHelper.psm1' -Resolve)
 
 #Helper function to build a markdown table.
 #Headers are an array of strings with format "label;alignment" where alignment is 'left', 'right' or 'center'
@@ -132,12 +133,14 @@ function GetTestResultSummaryMD {
 
     $summarySb = [System.Text.StringBuilder]::new()
     $failuresSb = [System.Text.StringBuilder]::new()
+    $totalTests = 0
+    $totalTime = 0.0
+    $totalFailed = 0
+    $totalSkipped = 0
+
     if (Test-Path -Path $testResultsFile -PathType Leaf) {
         $testResults = [xml](Get-Content -path $testResultsFile -Encoding UTF8)
-        $totalTests = 0
-        $totalTime = 0.0
-        $totalFailed = 0
-        $totalSkipped = 0
+
         if ($testResults.testsuites) {
             $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "appName" } | ForEach-Object { $_.Value } } | Select-Object -Unique)
             if (-not $appNames) {
@@ -219,6 +222,19 @@ function GetTestResultSummaryMD {
     else {
         $failuresSummaryMD = ''
     }
+
+    # Log test metrics to telemetry
+    $totalPassed = $totalTests - $totalFailed - $totalSkipped
+    if ($totalTests -gt 0) {
+        $telemetryData = [System.Collections.Generic.Dictionary[[System.String], [System.String]]]::new()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalTests' -Value $totalTests.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalPassed' -Value $totalPassed.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalFailed' -Value $totalFailed.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalSkipped' -Value $totalSkipped.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalTime' -Value $totalTime.ToString()
+        Trace-Information -Message "AL-Go Test Results - Tests" -AdditionalData $telemetryData
+    }
+
     $summarySb.ToString()
     $failuresSb.ToString()
     $failuresSummaryMD
@@ -320,6 +336,10 @@ function GetBcptSummaryMD {
     $lastCodeunitName = ''
     $lastOperationName = ''
 
+    $totalPassed = 0
+    $totalFailed = 0
+    $totalSkipped = 0
+
     # calculate statistics on measurements, skipping the $skipMeasurements longest measurements
     foreach($suiteName in $bcpt.Keys) {
         $suite = $bcpt."$suiteName"
@@ -418,6 +438,16 @@ function GetBcptSummaryMD {
                             }
                         }
                         $mdTableRow = @($thisSuiteName, $thisCodeunitID, $thisCodeunitName, $thisOperationName, $statusStr, $durationMinStr, $baseDurationMinStr, $diffDurationMinStr, $diffDurationMinPctStr, $numberOfSQLStmtsStr, $baseNumberOfSQLStmtsStr, $diffNumberOfSQLStmtsStr, $diffNumberOfSQLStmtsPctStr)
+
+                        # Update test counts
+                        if ($statusStr) {
+                            switch ($statusStr) {
+                                $statusOK { $totalPassed++ }
+                                $statusWarning { $totalFailed++ }
+                                $statusError { $totalFailed++ }
+                                $statusSkipped { $totalSkipped++ }
+                            }
+                        }
                     }
                 }
                 $mdTableRows.Add($mdTableRow) | Out-Null
@@ -438,6 +468,17 @@ function GetBcptSummaryMD {
         $summarySb.AppendLine("\n<i>No baseline provided. Copy a set of BCPT results to $([System.IO.Path]::GetFileName($baseLinePath)) in the project folder in order to establish a baseline.</i>") | Out-Null
     }
 
+    # Log BCPT metrics to telemetry
+    $totalTests = $totalPassed + $totalFailed + $totalSkipped
+    if ($totalTests -gt 0) {
+        $telemetryData = [System.Collections.Generic.Dictionary[[System.String], [System.String]]]::new()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalTests' -Value $totalTests.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalPassed' -Value $totalPassed.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalFailed' -Value $totalFailed.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalSkipped' -Value $totalSkipped.ToString()
+        Trace-Information -Message "AL-Go Test Results - BCPT Tests" -AdditionalData $telemetryData
+    }
+
     $summarySb.ToString()
 }
 
@@ -449,13 +490,13 @@ function GetPageScriptingTestResultSummaryMD {
 
     $summarySb = [System.Text.StringBuilder]::new()
     $failuresSb = [System.Text.StringBuilder]::new()
+    $totalTests = 0
+    $totalTime = 0.0
+    $totalFailed = 0
+    $totalSkipped = 0
 
     if (Test-Path -Path $testResultsFile -PathType Leaf) {
         $testResults = [xml](Get-Content -path $testResultsFile -Encoding UTF8)
-        $totalTests = 0
-        $totalTime = 0.0
-        $totalFailed = 0
-        $totalSkipped = 0
 
         $rootFailureNode = [FailureNode]::new($false)
         if ($testResults.testsuites) {
@@ -524,6 +565,17 @@ function GetPageScriptingTestResultSummaryMD {
         $failuresSummaryMD = ''
     }
 
+    # Log test metrics to telemetry
+    if ($totalTests -gt 0) {
+        $telemetryData = [System.Collections.Generic.Dictionary[[System.String], [System.String]]]::new()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalTests' -Value $totalTests.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalPassed' -Value $totalPassed.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalFailed' -Value $totalFailed.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalSkipped' -Value $totalSkipped.ToString()
+        Add-TelemetryProperty -Hashtable $telemetryData -Key 'TotalTime' -Value $totalTime.ToString()
+        Trace-Information -Message "AL-Go Test Results - Page scripting Tests" -AdditionalData $telemetryData
+    }
+
     return @{
         SummaryMD = $summarySb.ToString()
         FailuresMD = $failuresSb.ToString()

RELEASENOTES.md

@@ -44,6 +44,10 @@ Example using conditional settings to target specific workflows:
 
 Read more at [workflowDefaultInputs](https://aka.ms/algosettings#workflowDefaultInputs).
 
+## AL-Go Telemetry updates
+
+AL-Go telemetry now includes test results so you can more easily see how many AL tests, Page Scripting tests and BCPT tests ran in your workflows across all your repositories. Documentation for this can be found on [this article](https://github.com/microsoft/AL-Go/blob/main/Scenarios/EnablingTelemetry.md) on enabling telemetry.
+
 ### Issues
 
 - Issue 2039 Error when deploy to environment: NewTemporaryFolder is not recognized

Scenarios/EnablingTelemetry.md

@@ -86,6 +86,29 @@ traces
 | where message contains "AL-Go action"
 ```
 
+The following query gets all telemetry emitted for test results
+
+```
+traces
+| where timestamp > ago(7d)
+| project   timestamp,
+            message,
+            severityLevel,
+            RepositoryOwner = tostring(customDimensions.RepositoryOwner),
+            RepositoryName = tostring(customDimensions.RepositoryName),
+            RunId = tostring(customDimensions.RunId),
+            RunNumber = tostring(customDimensions.RunNumber),
+            RunAttempt = tostring(customDimensions.RunAttempt),
+            RefName = tostring(customDimensions.RefName),
+            TotalTests = todouble(customDimensions.TotalTests),
+            TotalFailed = todouble(customDimensions.TotalFailed),
+            TotalSkipped = todouble(customDimensions.TotalSkipped),
+            TotalPassed = todouble(customDimensions.TotalPassed),
+            TotalTime = todouble(customDimensions.TotalTime)
+| extend HtmlUrl = strcat("https://github.com/", RepositoryName, "/actions/runs/", RunId)
+| where message contains "AL-Go Test Results"
+```
+
 ## Telemetry events and data
 
 AL-Go logs four different types of telemetry events: AL-Go action ran/failed and AL-Go workflow ran/failed. Each of those telemetry events provide slightly different telemetry but common dimensions for all of them are:
@@ -159,6 +182,28 @@ Additional Dimensions:
 | RunsOn | Value of the RunsOn setting |
 | ALGoVersion | The AL-Go version used for the workflow run |
 
+### Test results
+
+Telemetry messages:
+
+- `AL-Go Test Results - Tests`
+- `AL-Go Test Results - Page scripting Tests`
+- `AL-Go Test Results - BCPT Tests`
+
+SeverityLevel: 1
+
+Additional Dimensions:
+
+| Dimension | Description |
+|-----------|-------------|
+| TotalTests | The total number of tests executed |
+| TotalFailed | The number of tests that failed |
+| TotalSkipped | The number of tests that were skipped |
+| TotalPassed | The number of tests that passed |
+| TotalTime | The total time taken to execute all tests |
+
+**Note:** The `TotalTime` dimension is not tracked for BCPT test results. For BCPT tests, only `TotalTests`, `TotalPassed`, `TotalFailed`, and `TotalSkipped` are available.
+
 ______________________________________________________________________
 
 [back](../README.md)