fix(evals): require rpi-agent baseline run when its named skills change

docs/contributing/evals-ci.md

@@ -3,7 +3,7 @@ title: Evals in CI
 description: Auth contract, fork-PR policy, and how to add a new eval spec for the hve-core vally pipeline
 sidebar_position: 11
 author: Microsoft
-ms.date: 2026-10-12
+ms.date: 2026-10-14
 ms.topic: how-to
 keywords:
   - evals
@@ -83,7 +83,11 @@ The execution matrix therefore contains one prompt row plus the planned agent,
 instruction, and skill rows. `max-parallel: 6` bounds that matrix. Baseline equivalence
 runs only when the canonical plan marks `baseline.required: true`; its fixed GPT and
 Claude producers use separate runner filesystems and a configurable maximum
-parallelism of two.
+parallelism of two. The plan requires it when the changed-artifact manifest lists
+`rpi-agent` in `affectedAgents`. A change reaches that list through the agent
+dependency map, or by touching any file inside a skill directory that the
+equivalence harness materializes for `rpi-agent`, including the skills it names in
+backticks such as `rpi-research`.
 
 At default shard counts the matrix holds more rows than the cap allows to start at
 once, so some rows wait for a free slot. To keep the longest producers from waiting

scripts/evals/Get-ChangedAIArtifact.ps1

@@ -38,6 +38,12 @@
 .PARAMETER RepoRoot
     Repository root. Defaults to the git toplevel or this script's parent directory.
 
+.PARAMETER EquivalenceSubject
+    Agents whose baseline-equivalence surface is checked against the changed paths.
+    A change inside a skill directory that the equivalence harness materializes for
+    one of these agents adds the agent to `affectedAgents`, even when the dependency
+    map does not record the skill. Defaults to `rpi-agent`.
+
 .EXAMPLE
     pwsh -File scripts/evals/Get-ChangedAIArtifact.ps1 -ChangeSetPath logs/eval-change-set.json
     Classify the frozen comparison and emit logs/changed-ai-artifacts.json.
@@ -62,7 +68,10 @@ param(
     [string]$OutFile,
 
     [Parameter(Mandatory = $false)]
-    [string]$RepoRoot
+    [string]$RepoRoot,
+
+    [Parameter(Mandatory = $false)]
+    [string[]]$EquivalenceSubject = @('rpi-agent')
 )
 
 Set-StrictMode -Version Latest
@@ -71,6 +80,7 @@ $ErrorActionPreference = 'Stop'
 Import-Module (Join-Path $PSScriptRoot 'Modules/ArtifactDetection.psm1') -Force
 Import-Module (Join-Path $PSScriptRoot 'Modules/AffectedAgents.psm1') -Force
 Import-Module (Join-Path $PSScriptRoot 'Modules/EvalChangeSet.psm1') -Force
+Import-Module (Join-Path $PSScriptRoot 'lib/EquivalenceEnvironment.psm1') -Force
 
 function Resolve-RepoRoot {
     [CmdletBinding()]
@@ -134,6 +144,70 @@ function Get-TrackedFileChangeSet {
     }
 }
 
+function Get-EquivalenceSurfaceAgent {
+    <#
+    .SYNOPSIS
+    Returns the equivalence subjects whose materialized skill surface a change touches.
+
+    .DESCRIPTION
+    The dependency map behind Get-AffectedAgentSlugs records only skills referenced
+    by path. The baseline-equivalence harness also resolves skills an agent names in
+    backticks, which is how the RPI agent references `rpi-research` and its peers, and
+    copies those whole skill directories into the surface it measures. Checking the
+    changed paths against the directories the harness materializes keeps the
+    equivalence gate and the harness in agreement. The check covers only the
+    equivalence subjects, not every agent in the dependency map.
+
+    .PARAMETER ChangedPath
+    Repository-relative changed paths, including the previous path of renames.
+
+    .PARAMETER Subject
+    Agent slugs whose equivalence surface is checked.
+
+    .PARAMETER RepoRoot
+    Repository root.
+
+    .OUTPUTS
+    [string[]] Subjects with at least one changed path inside their skill surface.
+    #>
+    [CmdletBinding()]
+    [OutputType([string[]])]
+    param(
+        [Parameter(Mandatory = $true)]
+        [AllowEmptyCollection()]
+        [string[]]$ChangedPath,
+
+        [Parameter(Mandatory = $true)]
+        [AllowEmptyCollection()]
+        [string[]]$Subject,
+
+        [Parameter(Mandatory = $true)]
+        [string]$RepoRoot
+    )
+
+    $agentsRoot = Join-Path $RepoRoot '.github/agents'
+    $touched = [System.Collections.Generic.List[string]]::new()
+    foreach ($slug in $Subject) {
+        if ([string]::IsNullOrWhiteSpace($slug)) { continue }
+        $agentFile = Get-ChildItem -LiteralPath $agentsRoot -Recurse -File -Filter "$slug.agent.md" -ErrorAction SilentlyContinue |
+            Select-Object -First 1
+        if (-not $agentFile) { continue }
+        $agentRelative = $agentFile.FullName.Substring($RepoRoot.Length).TrimStart('\', '/').Replace('\', '/')
+
+        $skillPrefixes = @(Get-AgentSurfaceSkill -RepoRoot $RepoRoot -AgentRelativePath $agentRelative | ForEach-Object { "$_/" })
+        foreach ($path in $ChangedPath) {
+            $normalized = ([string]$path -replace '\\', '/').TrimStart('/')
+            $inSurface = @($skillPrefixes | Where-Object { $normalized.StartsWith($_, [System.StringComparison]::OrdinalIgnoreCase) }).Count -gt 0
+            if ($inSurface) {
+                $touched.Add($slug)
+                break
+            }
+        }
+    }
+
+    return , [string[]]$touched.ToArray()
+}
+
 function Invoke-ChangedArtifactScan {
     <#
     .SYNOPSIS
@@ -148,6 +222,9 @@ function Invoke-ChangedArtifactScan {
     .PARAMETER RepoRoot
     Repository root.
 
+    .PARAMETER EquivalenceSubject
+    Agents whose baseline-equivalence skill surface is checked against the changes.
+
     .OUTPUTS
     [hashtable] `@{ baseRef; headRef; artifacts = @(...) }`.
     #>
@@ -161,7 +238,9 @@ function Invoke-ChangedArtifactScan {
         [switch]$AllTracked,
 
         [Parameter(Mandatory = $true)]
-        [string]$RepoRoot
+        [string]$RepoRoot,
+
+        [string[]]$EquivalenceSubject = @('rpi-agent')
     )
 
     if ($PSCmdlet.ParameterSetName -eq 'AllTracked') {
@@ -187,6 +266,8 @@ function Invoke-ChangedArtifactScan {
     $affectedAgents = [string[]]@()
     if ($changedPaths.Count -gt 0) {
         $affectedAgents = Get-AffectedAgentSlugs -ChangedFiles $changedPaths.ToArray() -RepoRoot $RepoRoot
+        $surfaceAgents = Get-EquivalenceSurfaceAgent -ChangedPath $changedPaths.ToArray() -Subject $EquivalenceSubject -RepoRoot $RepoRoot
+        $affectedAgents = [string[]]@(@($affectedAgents) + @($surfaceAgents) | Sort-Object -Unique)
     }
 
     return @{
@@ -209,10 +290,10 @@ if ($MyInvocation.InvocationName -ne '.') {
 
     try {
         $manifest = if ($AllTracked) {
-            Invoke-ChangedArtifactScan -AllTracked -RepoRoot $resolvedRepoRoot
+            Invoke-ChangedArtifactScan -AllTracked -RepoRoot $resolvedRepoRoot -EquivalenceSubject $EquivalenceSubject
         }
         else {
-            Invoke-ChangedArtifactScan -ChangeSetPath $ChangeSetPath -RepoRoot $resolvedRepoRoot
+            Invoke-ChangedArtifactScan -ChangeSetPath $ChangeSetPath -RepoRoot $resolvedRepoRoot -EquivalenceSubject $EquivalenceSubject
         }
     }
     catch {

scripts/evals/lib/EquivalenceEnvironment.psm1

@@ -342,6 +342,52 @@ function Get-AgentDeclaredDependency {
     }
 }
 
+function Get-AgentSurfaceSkill {
+    <#
+    .SYNOPSIS
+        Returns the skill directories materialized for an agent's customization surface.
+    .DESCRIPTION
+        The surface carries the skills the agent references and, transitively, the skills
+        its declared subagents reference, because a subagent's skills are part of what the
+        parent agent delivers. `New-CustomizedEnvironment` copies exactly these
+        directories, and the change gate in `Get-ChangedAIArtifact.ps1` checks changed
+        paths against the same set, so the gate and the harness cannot disagree about
+        which skills an agent depends on.
+    .OUTPUTS
+        [string[]] Workspace-relative skill directory paths, deduplicated and sorted.
+    .PARAMETER RepoRoot
+        Absolute path to the repository root that references are resolved against.
+    .PARAMETER AgentRelativePath
+        Workspace-relative path to the agent file whose surface is resolved.
+    #>
+    [CmdletBinding()]
+    [OutputType([string[]])]
+    param(
+        [Parameter(Mandatory = $true)]
+        [ValidateNotNullOrEmpty()]
+        [string]$RepoRoot,
+
+        [Parameter(Mandatory = $true)]
+        [ValidateNotNullOrEmpty()]
+        [string]$AgentRelativePath
+    )
+
+    $Declared = Get-AgentDeclaredDependency -RepoRoot $RepoRoot -AgentRelativePath $AgentRelativePath
+
+    $SkillDirs = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
+    foreach ($Relative in (Get-AgentSkillReference -RepoRoot $RepoRoot -AgentFilePath $AgentRelativePath)) {
+        [void]$SkillDirs.Add($Relative)
+    }
+    foreach ($SubagentRelative in @($Declared.Subagents)) {
+        if ([string]::IsNullOrWhiteSpace($SubagentRelative)) { continue }
+        foreach ($Relative in (Get-AgentSkillReference -RepoRoot $RepoRoot -AgentFilePath ([string]$SubagentRelative))) {
+            [void]$SkillDirs.Add($Relative)
+        }
+    }
+
+    return @($SkillDirs) | Sort-Object
+}
+
 function Copy-VerifiedRepositoryFile {
     <#
     .SYNOPSIS
@@ -468,20 +514,9 @@ function New-CustomizedEnvironment {
         $Applied.Add($CopilotInstructions)
     }
 
-    $SkillDirs = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
-    foreach ($Relative in (Get-AgentSkillReference -RepoRoot $RepoRoot -AgentFilePath $AgentRelative)) {
-        [void]$SkillDirs.Add($Relative)
-    }
     # A subagent's skills are part of the surface the parent agent delivers, so they
     # are resolved transitively rather than left to the agent body alone.
-    foreach ($SubagentRelative in @($Declared.Subagents)) {
-        if ([string]::IsNullOrWhiteSpace($SubagentRelative)) { continue }
-        foreach ($Relative in (Get-AgentSkillReference -RepoRoot $RepoRoot -AgentFilePath ([string]$SubagentRelative))) {
-            [void]$SkillDirs.Add($Relative)
-        }
-    }
-
-    foreach ($Relative in ($SkillDirs | Sort-Object)) {
+    foreach ($Relative in (Get-AgentSurfaceSkill -RepoRoot $RepoRoot -AgentRelativePath $AgentRelative)) {
         $Source = Join-Path $RepoRoot $Relative
         if (-not (Test-Path -LiteralPath $Source -PathType Container)) { continue }
         $null = Assert-ContainedRepositoryPath -Path $Source -ApprovedRoot $RepoRoot -Because 'skill directory'
@@ -723,6 +758,7 @@ function Save-BaselineCacheEntry {
 Export-ModuleMember -Function @(
     'Get-AgentSkillReference',
     'Get-AgentDeclaredDependency',
+    'Get-AgentSurfaceSkill',
     'Assert-ContainedRepositoryPath',
     'Resolve-AgentScopePattern',
     'New-CustomizedEnvironment',

scripts/tests/evals/EquivalenceEnvironment.Tests.ps1

@@ -148,6 +148,51 @@ Follow #file:../../instructions/sample/house.instructions.md and use `skill-one`
     }
 }
 
+Describe 'Get-AgentSurfaceSkill' -Tag 'Unit' {
+    BeforeAll {
+        $script:SurfaceRoot = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())
+        New-AgentFixture -Root $script:SurfaceRoot
+
+        $agentsDir = Join-Path $script:SurfaceRoot '.github/agents/sample'
+        New-Item -ItemType Directory -Path (Join-Path $agentsDir 'subagents') -Force | Out-Null
+        Set-Content -LiteralPath (Join-Path $agentsDir 'subagents/helper.agent.md') -Encoding UTF8 -Value @'
+---
+name: helper
+---
+
+Uses `skill-three` for its work.
+'@
+        Set-Content -LiteralPath (Join-Path $agentsDir 'agent-parent.agent.md') -Encoding UTF8 -Value @'
+---
+name: Agent Parent
+agents:
+  - helper
+---
+
+Use `skill-one`.
+'@
+    }
+
+    AfterAll {
+        Remove-Item -LiteralPath $script:SurfaceRoot -Recurse -Force -ErrorAction SilentlyContinue
+    }
+
+    It 'Includes skills named by the agent and by its declared subagents' {
+        $skills = @(Get-AgentSurfaceSkill -RepoRoot $script:SurfaceRoot -AgentRelativePath '.github/agents/sample/agent-parent.agent.md')
+        $skills | Should -BeExactly @('.github/skills/alpha/skill-one', '.github/skills/beta/skill-three')
+    }
+
+    It 'Matches the skill directories the customized environment materializes' {
+        $workspace = Join-Path $TestDrive ('surface-ws-' + [Guid]::NewGuid().ToString('N'))
+        $skillDir = Join-Path $TestDrive ('surface-skills-' + [Guid]::NewGuid().ToString('N'))
+        $result = New-CustomizedEnvironment -RepoRoot $script:SurfaceRoot -Agent 'agent-parent' -WorkspacePath $workspace -SkillDirPath $skillDir
+
+        $materialized = @($result.Applied | Where-Object { $_ -like '*/SKILL.md' } | ForEach-Object { $_ -replace '/SKILL\.md$', '' })
+        $surface = @(Get-AgentSurfaceSkill -RepoRoot $script:SurfaceRoot -AgentRelativePath '.github/agents/sample/agent-parent.agent.md')
+        $materialized | Should -BeExactly $surface
+    }
+}
+
 Describe 'Assert-ContainedRepositoryPath' -Tag 'Unit' {
     BeforeAll {
         $script:GuardRoot = Join-Path $TestDrive ('guard-' + [Guid]::NewGuid().ToString('N'))
@@ -562,6 +607,13 @@ Describe 'Repository agents' -Tag 'Unit' {
         $doc.Count | Should -BeGreaterThan 0
         ($rpi -join ',') | Should -Not -Be ($doc -join ',')
     }
+
+    It 'Resolves the RPI skills named in backticks as the rpi-agent surface' {
+        $surface = @(Get-AgentSurfaceSkill -RepoRoot $script:RepoRoot -AgentRelativePath '.github/agents/hve-core/rpi-agent.agent.md')
+        foreach ($skill in @('rpi-research', 'rpi-plan', 'rpi-implement', 'rpi-review')) {
+            $surface | Should -Contain ".github/skills/rpi/$skill"
+        }
+    }
 }
 
 Describe 'Resolve-AgentScopePattern' -Tag 'Unit' {

scripts/tests/evals/Get-ChangedAIArtifact.Tests.ps1

@@ -265,6 +265,54 @@ Describe 'AI artifact canonical input' -Tag 'Unit' {
     }
 }
 
+Describe 'Equivalence surface gate' -Tag 'Unit' {
+    BeforeAll {
+        # The subject names its skill in backticks, as the real RPI agent does, so the
+        # path-only dependency map records no skill for it.
+        $script:SurfaceRepo = Join-Path $TestDrive ('surface-' + [Guid]::NewGuid().ToString('N'))
+        $agentDir = Join-Path $script:SurfaceRepo '.github/agents/hve-core'
+        $null = New-Item -ItemType Directory -Path $agentDir -Force
+        Set-Content -LiteralPath (Join-Path $agentDir 'rpi-agent.agent.md') -Value "---`nname: RPI Agent`n---`n`nActivate ``rpi-research`` first.`n"
+        foreach ($skill in @('rpi/rpi-research', 'shared/unrelated')) {
+            $skillDir = Join-Path $script:SurfaceRepo ".github/skills/$skill"
+            $null = New-Item -ItemType Directory -Path (Join-Path $skillDir 'references') -Force
+            Set-Content -LiteralPath (Join-Path $skillDir 'SKILL.md') -Value 'skill'
+            Set-Content -LiteralPath (Join-Path $skillDir 'references/notes.md') -Value 'notes'
+        }
+
+        $script:ScanChange = {
+            param([string]$Path, [string[]]$Subject = @('rpi-agent'))
+
+            $changeSetPath = Join-Path $TestDrive ('surface-change-' + [Guid]::NewGuid().ToString('N') + '.json')
+            @{
+                schemaVersion = '1.0'; baseRef = 'a' * 40; headRef = 'b' * 40; comparisonBase = 'c' * 40
+                changes = @(@{ status = 'M'; path = $Path; previousPath = $null })
+            } | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $changeSetPath
+            return Invoke-ChangedArtifactScan -ChangeSetPath $changeSetPath -RepoRoot $script:SurfaceRepo -EquivalenceSubject $Subject
+        }
+    }
+
+    It 'marks the subject affected when a backtick-named skill changes' {
+        $result = & $script:ScanChange '.github/skills/rpi/rpi-research/SKILL.md'
+        @($result.affectedAgents) | Should -BeExactly @('rpi-agent')
+    }
+
+    It 'marks the subject affected when a file inside that skill directory changes' {
+        $result = & $script:ScanChange '.github/skills/rpi/rpi-research/references/notes.md'
+        @($result.affectedAgents) | Should -BeExactly @('rpi-agent')
+    }
+
+    It 'leaves the subject unaffected by a skill outside its surface' {
+        $result = & $script:ScanChange '.github/skills/shared/unrelated/SKILL.md'
+        $result.affectedAgents | Should -HaveCount 0
+    }
+
+    It 'checks only the configured equivalence subjects' {
+        $result = & $script:ScanChange '.github/skills/rpi/rpi-research/SKILL.md' @()
+        $result.affectedAgents | Should -HaveCount 0
+    }
+}
+
 Describe 'AI artifact full-scope input' -Tag 'Unit' {
     BeforeAll {
         $script:FullScopeRepo = Join-Path $TestDrive ('full-' + [Guid]::NewGuid())