test(dashboard): run Playwright tests as non-blocking outerloop

.github/workflows/filter-nonblocking-test-artifacts.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the MIT license.
+#
+# Filters downloaded outerloop test artifacts before the scheduled failure
+# reporter enumerates gating failures.
+#
+# Input: one directory containing downloaded artifacts in this shape:
+#   <all-logs>/logs-*/testresults/ignore-test-failures.marker
+# Output: removes each marked logs-* artifact directory and writes the removed
+# paths to stdout. It does not create an output file.
+# Failure: exits nonzero for invalid arguments, a missing input directory, or a
+# marker outside the expected artifact layout. All markers are validated before
+# any directory is removed so an unexpected layout cannot partially filter the
+# results.
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 <all-logs-directory>" >&2
+  exit 1
+fi
+
+all_logs_directory="${1%/}"
+
+if [[ ! -d "$all_logs_directory" ]]; then
+  echo "Test artifact directory does not exist: $all_logs_directory" >&2
+  exit 1
+fi
+
+marker_list="$(mktemp)"
+trap 'rm -f "$marker_list"' EXIT
+find "$all_logs_directory" -type f -name ignore-test-failures.marker -print0 > "$marker_list"
+
+# Validate every marker before deleting anything so an unexpected artifact
+# layout fails closed without partially filtering the downloaded results.
+while IFS= read -r -d '' marker; do
+  artifact_directory="$(dirname "$(dirname "$marker")")"
+  artifact_parent_directory="$(dirname "$artifact_directory")"
+  artifact_name="$(basename "$artifact_directory")"
+  expected_marker="$artifact_directory/testresults/ignore-test-failures.marker"
+  if [[ "$artifact_parent_directory" != "$all_logs_directory" ||
+        "$artifact_name" != logs-* ||
+        "$marker" != "$expected_marker" ]]; then
+    echo "Unexpected ignore marker location: $marker" >&2
+    exit 1
+  fi
+done < "$marker_list"
+
+while IFS= read -r -d '' marker; do
+  artifact_directory="$(dirname "$(dirname "$marker")")"
+  echo "Excluding non-gating test results from failure classification: $artifact_directory"
+  rm -rf -- "$artifact_directory"
+done < "$marker_list"

.github/workflows/native-dashboard-validation.yml

@@ -37,16 +37,19 @@ jobs:
 
       - name: Extract and smoke test Native AOT Dashboard
         shell: pwsh
+        env:
+          ARTIFACTS_DIRECTORY: ${{ github.workspace }}/artifacts/native-dashboard
+          RID: ${{ inputs.rid }}
         run: |
           $ErrorActionPreference = 'Stop'
-          $archive = @(Get-ChildItem -Path '${{ github.workspace }}/artifacts/native-dashboard' -Filter 'aspire-dashboard-${{ inputs.rid }}.zip' -File -Recurse)
+          $archive = @(Get-ChildItem -Path $env:ARTIFACTS_DIRECTORY -Filter "aspire-dashboard-$env:RID.zip" -File -Recurse)
           if ($archive.Count -ne 1) {
-            throw "Expected one Native AOT Dashboard archive for ${{ inputs.rid }}, found $($archive.Count): $($archive.FullName -join ', ')"
+            throw "Expected one Native AOT Dashboard archive for $env:RID, found $($archive.Count): $($archive.FullName -join ', ')"
           }
 
-          $extractDirectory = '${{ github.workspace }}/artifacts/native-dashboard/extracted'
+          $extractDirectory = Join-Path $env:ARTIFACTS_DIRECTORY 'extracted'
           New-Item -ItemType Directory -Path $extractDirectory | Out-Null
-          if ('${{ runner.os }}' -eq 'Windows') {
+          if ($env:RUNNER_OS -eq 'Windows') {
             Expand-Archive -LiteralPath $archive[0].FullName -DestinationPath $extractDirectory
             $dashboardPath = Join-Path $extractDirectory 'Aspire.Dashboard.exe'
           }
@@ -61,19 +64,40 @@ jobs:
           Add-Content -LiteralPath $env:GITHUB_ENV -Value "ASPIRE_NATIVE_DASHBOARD_PATH=$dashboardPath"
           ./eng/scripts/test-native-dashboard.ps1 -DashboardPath $dashboardPath
 
+      - name: Test Native AOT Dashboard
+        if: ${{ inputs.rid == 'win-x64' }}
+        shell: pwsh
+        env:
+          CONFIGURATION: ${{ inputs.configuration }}
+          NUGET_PACKAGES: ${{ format('{0}/.nuget/packages', github.workspace) }}
+        run: |
+          & ./dotnet.cmd test `
+            --project tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj `
+            -c $env:CONFIGURATION `
+            --no-launch-profile `
+            /p:ContinuousIntegrationBuild=true `
+            /p:RunOuterloopTests=true `
+            -- `
+            --filter-method "*.NativeDashboard_*" `
+            --filter-not-trait "quarantined=true"
+
+          $testExitCode = $LASTEXITCODE
+          & .github/workflows/normalize-mtp-exit-code.ps1 -ExitCode $testExitCode
+
       - name: Test Native AOT Dashboard interactivity
         if: ${{ inputs.rid == 'win-x64' }}
         shell: pwsh
         env:
+          CONFIGURATION: ${{ inputs.configuration }}
           NUGET_PACKAGES: ${{ format('{0}/.nuget/packages', github.workspace) }}
         run: >
           ./dotnet.cmd test
-          --project tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj
-          -c ${{ inputs.configuration }}
+          --project tests/Aspire.Dashboard.Playwright.Tests/Aspire.Dashboard.Playwright.Tests.csproj
+          -c $env:CONFIGURATION
           --no-launch-profile
           /p:ContinuousIntegrationBuild=true
           /p:InstallBrowsersForPlaywright=true
           /p:RunOuterloopTests=true
           --
-          --filter-method "*.NativeDashboard_LoadsInteractivePageWithoutBrowserErrors"
-          --filter-not-trait "quarantined=true"
+          --filter-method "*.NativeDashboard_*"
+          --filter-not-trait "quarantined=true"

.github/workflows/report-specialized-test-failures.js

@@ -8,8 +8,8 @@
 // auto-close-on-green).
 //
 // Two failure kinds:
-//   - 'test-failures' : outerloop tests failed. Each failed run's comment lists
-//     the failing tests; a dev can split them into per-test issues later.
+//   - 'test-failures' : gating outerloop tests failed. Each failed run's comment
+//     lists the failing tests; a dev can split them into per-test issues later.
 //   - 'infra'         : the run broke before/around test execution (build/setup,
 //     missing TRX). Quarantine runs swallow test failures (ignoreTestFailures),
 //     so a *failed* quarantine run is always infra.

.github/workflows/run-tests.yml

@@ -34,11 +34,18 @@ on:
         required: false
         type: string
         default: "ubuntu-latest"
-      # When true, ignores test failures and checks if .trx files have nonzero test count
+      # When true, known test-runner outcomes do not fail the job. Unknown or
+      # infrastructure/configuration failures remain blocking.
       ignoreTestFailures:
         required: false
         type: boolean
         default: false
+      # Specialized workflows can legitimately produce zero tests for a project/OS
+      # combination after applying their trait filter.
+      allowZeroTests:
+        required: false
+        type: boolean
+        default: false
 
 jobs:
 
@@ -427,14 +434,14 @@ jobs:
             ${{ inputs.extraTestArgs }} || TEST_EXIT_CODE=$?
 
           # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome
-          # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode.
+          # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored.
           echo "$TEST_EXIT_CODE" > "${{ github.workspace }}/test-exit-code.txt"
 
           # Stop heartbeat monitor
           kill $HEARTBEAT_PID 2>/dev/null || true
           wait $HEARTBEAT_PID 2>/dev/null || true
 
-          # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job
+          # When ignoreTestFailures is true a known test outcome must not fail the job
           # here; the authoritative pass/fail decision is made later by "Verify test results exist".
           if [ "${{ inputs.ignoreTestFailures }}" = "true" ]; then
             if [ "$TEST_EXIT_CODE" -ne 0 ]; then
@@ -490,10 +497,10 @@ jobs:
           }
 
           # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome
-          # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode.
+          # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored.
           Set-Content -Path "${{ github.workspace }}/test-exit-code.txt" -Value $testExitCode
 
-          # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job
+          # When ignoreTestFailures is true a known test outcome must not fail the job
           # here. The Linux/macOS step neutralizes the exit code inline; this pwsh step captures
           # $testExitCode instead, so it must explicitly exit 0. Otherwise the non-zero $LASTEXITCODE
           # survives to the end of the script and GitHub's pwsh wrapper — which appends
@@ -555,14 +562,14 @@ jobs:
           TEST_EXIT_CODE=$NORMALIZED_TEST_EXIT_CODE
 
           # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome
-          # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode.
+          # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored.
           echo "$TEST_EXIT_CODE" > "${{ github.workspace }}/test-exit-code.txt"
 
           # Stop heartbeat monitor
           kill $HEARTBEAT_PID 2>/dev/null || true
           wait $HEARTBEAT_PID 2>/dev/null || true
 
-          # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job
+          # When ignoreTestFailures is true a known test outcome must not fail the job
           # here; the authoritative pass/fail decision is made later by "Verify test results exist".
           if [ "${{ inputs.ignoreTestFailures }}" = "true" ]; then
             if [ "$TEST_EXIT_CODE" -ne 0 ]; then
@@ -630,10 +637,10 @@ jobs:
           }
 
           # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome
-          # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode.
+          # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored.
           Set-Content -Path "${{ github.workspace }}/test-exit-code.txt" -Value $testExitCode
 
-          # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job
+          # When ignoreTestFailures is true a known test outcome must not fail the job
           # here. The Linux/macOS step neutralizes the exit code inline; this pwsh step captures
           # $testExitCode instead, so it must explicitly exit 0. Otherwise the non-zero $LASTEXITCODE
           # survives to the end of the script and GitHub's pwsh wrapper — which appends
@@ -648,87 +655,30 @@ jobs:
 
           exit $testExitCode
 
+      - name: Mark ignored test results
+        if: ${{ always() && inputs.ignoreTestFailures }}
+        shell: pwsh
+        env:
+          TEST_SHORT_NAME: ${{ inputs.testShortName }}
+        run: |
+          $testResultsDir = "${{ github.workspace }}/testresults"
+          New-Item -Path $testResultsDir -ItemType Directory -Force | Out-Null
+          Set-Content -Path "$testResultsDir/ignore-test-failures.marker" -Value $env:TEST_SHORT_NAME
+
       - name: Verify test results exist
         if: ${{ inputs.ignoreTestFailures }}
         shell: pwsh
+        env:
+          ALLOW_ZERO_TESTS: ${{ inputs.allowZeroTests }}
         run: |
-          # Classify the test run using the MTP exit code captured by the test step.
-          # https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes
-          # In quarantine mode (ignoreTestFailures) we tolerate genuine test outcomes — 2 (a test failed),
-          # 3 (session aborted, e.g. a hangdump timeout), 7 (the test host crashed) and 13 (--maximum-failed-tests
-          # reached) — because quarantined tests are flaky by definition. But a broken harness must still fail
-          # the job, otherwise the quarantine run stays permanently green and hides real breakage:
-          #   4  = invalid extension setup
-          #   5  = invalid command-line arguments
-          #   10 = test adapter infrastructure failure (e.g. a fixture could not be created)
-          #   11 = the test process exited because a dependent process exited
-          #   12 = no supported protocol version
-          # (Exit code 8 "zero tests ran" is mapped to 0 either by --ignore-exit-code 8 for direct assembly
-          # execution or explicitly by the dotnet test steps above, so a genuinely empty run is caught below
-          # by the .trx/artifact checks rather than by the exit code.)
-          $exitCodeFile = "${{ github.workspace }}/test-exit-code.txt"
-          if (Test-Path $exitCodeFile) {
-            $mtpExitCode = [int]((Get-Content -Path $exitCodeFile -Raw).Trim())
-            $fatalInfraCodes = @(4, 5, 10, 11, 12)
-            if ($fatalInfraCodes -contains $mtpExitCode) {
-              Write-Host "::error::Test runner exited with code $mtpExitCode, which indicates a test infrastructure/configuration failure rather than a test failure. Failing the job even though ignoreTestFailures is true."
-              exit 1
-            }
-            Write-Host "Test runner exit code: $mtpExitCode (treated as a test outcome, not an infrastructure failure)"
-          } else {
-            Write-Warning "No test exit code file found at $exitCodeFile; relying on .trx/artifact checks only."
+          $arguments = @{
+            TestResultsPath = "${{ github.workspace }}/testresults"
+            ExitCodePath = "${{ github.workspace }}/test-exit-code.txt"
           }
-
-          $trxFiles = Get-ChildItem -Path "${{ github.workspace }}/testresults" -Filter *.trx -Recurse -ErrorAction SilentlyContinue
-
-          if ($trxFiles.Count -eq 0) {
-            # MTP does not produce .trx files when the test run is aborted (e.g., by --hangdump-timeout).
-            # Check for recordings or other test artifacts that prove tests actually ran before failing.
-            $hasRecordings = Test-Path "${{ github.workspace }}/testresults/recordings/*.cast"
-            $hasHangDumps = Get-ChildItem -Path "${{ github.workspace }}/testresults" -Filter *hangdump* -Recurse -ErrorAction SilentlyContinue
-            if ($hasRecordings -or $hasHangDumps) {
-              Write-Warning "No .trx files found, but test artifacts exist — test run likely aborted by timeout before trx was flushed."
-              exit 0
-            }
-            Write-Error "No .trx files found. Tests may not have run due to infrastructure issues."
-            exit 1
-          }
-
-          Write-Host "Found $($trxFiles.Count) .trx file(s)"
-
-          $validFileCount = 0
-          foreach ($trxFile in $trxFiles) {
-            Write-Host "Checking $($trxFile.Name)..."
-
-            try {
-              [xml]$trxContent = Get-Content -Path $trxFile.FullName
-
-              # Check if the XML is valid and non-empty
-              if ($trxContent -and $trxContent.TestRun) {
-                $validFileCount++
-                $countersNode = $trxContent.TestRun.ResultSummary.Counters
-
-                if ($countersNode -and $countersNode.total) {
-                  $testCount = [int]$countersNode.total
-                  Write-Host "  Total tests in file: $testCount"
-                } else {
-                  Write-Host "  File is valid but has no test count information"
-                }
-              } else {
-                Write-Warning "  File appears to be empty or invalid"
-              }
-            }
-            catch {
-              Write-Warning "Failed to parse $($trxFile.Name): $_"
-            }
+          if ([System.Convert]::ToBoolean($env:ALLOW_ZERO_TESTS)) {
+            $arguments.AllowZeroTests = $true
           }
-
-          if ($validFileCount -eq 0) {
-            Write-Error "No valid .trx files found. All .trx files are empty or invalid XML."
-            exit 1
-          }
-
-          Write-Host "✓ Test execution completed successfully (found $validFileCount valid .trx file(s))"
+          ./eng/scripts/verify-nonblocking-test-results.ps1 @arguments
 
       - name: Check for hang dump files
         if: always()
@@ -745,13 +695,8 @@ jobs:
           # even when all tests passed, so we don't fail on those.
           $hangDumpFiles = Get-ChildItem -Path $testResultsDir -Filter *hangdump* -Recurse -ErrorAction SilentlyContinue
           if ($hangDumpFiles.Count -gt 0) {
-            # When ignoreTestFailures is true (the quarantine workflow), a hang dump must NOT fail the job.
-            # Quarantined tests are flaky by definition and routinely time out; the whole point of that
-            # workflow is to surface flakiness without going red, so failing here defeats its purpose and
-            # leaves the quarantine run permanently broken. Downgrade to a warning instead — the dump is
-            # still detected, logged, and uploaded as an artifact for investigation.
-            # For all other callers (normal CI, outerloop) ignoreTestFailures is false, so a hang dump
-            # still fails the job: it indicates a real timeout that must not be ignored.
+            # A caller that explicitly ignores test outcomes also treats a test timeout as non-gating.
+            # The dump remains visible in the logs and uploaded artifacts. All other callers fail here.
             if ('${{ inputs.ignoreTestFailures }}' -eq 'true') {
               Write-Host "::warning::Hang dump file(s) detected — a test timed out (ignored because ignoreTestFailures is true):"
               foreach ($df in $hangDumpFiles) { Write-Host "  - $($df.FullName)" }

.github/workflows/specialized-test-runner.yml

@@ -22,11 +22,16 @@ on:
       enablePlaywrightInstall:
         required: true
         type: boolean
-      # When true, ignores test failures and checks if .trx files have nonzero test count
+      # When true, known test outcomes are non-blocking while infrastructure failures remain blocking.
       ignoreTestFailures:
         required: false
         type: boolean
         default: false
+      # Allows valid zero-test results for trait-filtered project/OS lanes where no tests match.
+      allowZeroTests:
+        required: false
+        type: boolean
+        default: false
 
 jobs:
 
@@ -160,7 +165,8 @@ jobs:
       os: ${{ matrix.tests.os }}
       mtpBaseArgs: ${{ matrix.tests.mtpBaseArgs }}
       extraTestArgs: ${{ inputs.extraTestArgs }}
-      ignoreTestFailures: ${{ inputs.ignoreTestFailures }}
+      ignoreTestFailures: ${{ inputs.ignoreTestFailures || matrix.tests.properties.nonBlockingTestFailures }}
+      allowZeroTests: ${{ inputs.allowZeroTests }}
 
 
   results: