Phase 3: canvas smoke-test evidence and weekly review metrics (#4184)

.github/auto-merge.yml

@@ -0,0 +1,109 @@
+# Safe auto-merge policy (github/awesome-copilot#4184, phase 3).
+#
+# Evaluated by .github/workflows/auto-merge.yml via eng/auto-merge.mjs.
+# See docs/maintainers/auto-merge-and-metrics.md for the full policy.
+#
+# While `enabled` is false the workflow only evaluates pull requests and writes
+# a dry-run report to the job summary. It never arms or disarms auto-merge.
+enabled: false
+
+# Branch that auto-merge may target.
+base_branch: main
+
+# Merge method passed to enablePullRequestAutoMerge: squash, merge, or rebase.
+merge_method: squash
+
+# Check runs (or commit statuses) that must have succeeded on the PR head SHA.
+required_checks:
+  - submission-gate
+
+# Every label listed here must be present.
+required_labels:
+  - merge-risk:low
+
+# Any of these labels blocks arming and disarms auto-merge that was armed by
+# this automation.
+blocking_labels:
+  - do-not-merge
+  - requires-submitter-fixes
+  - awaiting-automation
+  - merge-risk:medium
+  - merge-risk:high
+  - needs-discussion
+  - rejected
+
+# Minimum approving reviews from users other than the author. Reviews from bots
+# never count. Any outstanding "changes requested" review blocks auto-merge.
+min_approvals: 1
+
+# All review threads must be resolved.
+require_resolved_threads: true
+
+# The PR head must contain the latest base branch commit.
+require_up_to_date: true
+
+# Label added when this automation arms auto-merge. It records ownership so the
+# automation only disarms auto-merge that it armed itself.
+armed_label: auto-merge-armed
+
+# Disarm auto-merge enabled by someone else (a maintainer) when a blocking
+# label is added. Other failed conditions never touch maintainer-armed merges.
+disarm_manual_on_blocking_label: true
+
+# Post (and update) a short status comment when auto-merge is armed/disarmed.
+comment: true
+
+# Initial allowlist: a PR must match at least one of these eligibility rules
+# in addition to every condition above.
+eligibility:
+  # PRs that only change generated output, opened by automation accounts.
+  generated_output:
+    enabled: true
+    # Logins are compared case-insensitively with any `[bot]` suffix or `app/`
+    # prefix removed.
+    authors:
+      - github-actions
+      - allcontributors
+    paths:
+      - README.md
+      - docs/README.*.md
+      - .all-contributorsrc
+      - .github/plugin/marketplace.json
+      - plugins/external.json
+
+  # Low-risk updates to existing resources from their established owners.
+  resource_owner:
+    enabled: true
+    # Only modifications of existing files qualify (no additions, deletions,
+    # or renames).
+    modified_only: true
+    # Owner sources: CODEOWNERS entries for the touched path (the catch-all `*`
+    # rule is ignored) and the resource's recorded author (plugin.json
+    # author.url / author.name, or the author of the commit that added the
+    # resource).
+    sources:
+      - codeowners
+      - recorded_author
+    # The author must already have at least this many merged PRs in the repo.
+    min_merged_prs: 1
+    # Top-level resource directories an owner update may touch.
+    resource_roots:
+      - agents
+      - instructions
+      - skills
+      - hooks
+      - workflows
+      - plugins
+      - extensions
+    # Generated files that may change alongside the resource (from
+    # `npm run build`) without needing an owner of their own.
+    generated_paths:
+      - README.md
+      - docs/README.*.md
+      - .github/plugin/marketplace.json
+    # Any change to these paths makes the PR ineligible for owner auto-merge.
+    excluded_paths:
+      - plugins/external.json
+      - .github/**
+      - eng/**
+      - CODEOWNERS

.github/review-metrics.yml

@@ -0,0 +1,62 @@
+# Weekly review operating metrics (github/awesome-copilot#4184, phase 3).
+#
+# Read by eng/review-metrics.mjs, which .github/workflows/review-metrics.yml
+# runs every Monday. See docs/maintainers/auto-merge-and-metrics.md.
+
+# Days of activity summarized in each report.
+window_days: 7
+
+# Review targets, in business days (weekends are skipped).
+targets_business_days: [2, 4]
+
+# Base branch for contribution PRs.
+base_branch: main
+
+# State labels (maintained by the submission gate). Open PRs without any of
+# these are reported as "unlabeled".
+state_labels:
+  - awaiting-automation
+  - requires-submitter-fixes
+  - ready-for-review
+  - review-in-progress
+  - approved
+
+# Risk tier labels. Open PRs without any of these are "unclassified".
+risk_labels:
+  - merge-risk:low
+  - merge-risk:medium
+  - merge-risk:high
+
+# External plugin submission issues are tracked alongside PRs.
+external_plugin_label: external-plugin
+external_plugin_state_labels:
+  - awaiting-review
+  - requires-submitter-fixes
+  - ready-for-review
+  - awaiting-approval
+
+# Review-automation workflows whose failed/errored runs count toward the
+# automation failure rate. Files that do not exist yet are reported as
+# "not found" and skipped, so later phases can be listed ahead of time.
+automation_workflows:
+  - submission-gate.yml
+  - submission-gate-writer.yml
+  - review-routing.yml
+  - validate-canvas-extensions.yml
+  - canvas-smoke-test-comment.yml
+  - auto-merge.yml
+  - pr-risk-scan.yml
+  - pr-risk-scan-comment.yml
+  - label-pr-intent.yml
+  - label-pr-intent-writer.yml
+  - check-plugin-structure.yml
+  - external-plugin-intake.yml
+  - external-plugin-quality-gates.yml
+  - external-plugin-pr-quality-gates.yml
+  - external-plugin-pr-quality-gates-writer.yml
+  - external-plugin-command-router.yml
+
+# Tracking issue that receives the weekly report.
+tracking_issue:
+  title: Review operating metrics
+  label: review-metrics

.github/workflows/auto-merge.yml

@@ -0,0 +1,104 @@
+name: Safe Auto-Merge
+
+# Arms GitHub auto-merge for allowlisted low-risk PRs once every policy
+# condition in .github/auto-merge.yml holds, and disarms it when conditions
+# later fail. Evaluation uses API metadata only: this workflow checks out the
+# base branch and never checks out or executes pull request code.
+#
+# Policy: docs/maintainers/auto-merge-and-metrics.md
+
+on:
+  pull_request_target:
+    types: [opened, reopened, synchronize, ready_for_review, converted_to_draft, labeled, unlabeled]
+    branches: [main]
+  check_run:
+    types: [completed]
+  workflow_run:
+    # The Phase 2 writer applies merge-risk:* and state labels; labels are final
+    # once it completes. Its event does not identify a PR, so this sweeps.
+    workflows: ["Submission Gate Writer"]
+    types: [completed]
+  schedule:
+    # Periodic sweep: picks up approvals and resolved threads (review events
+    # are not used because they run with the PR's workflow definition), label
+    # or check changes made with GITHUB_TOKEN (which do not trigger workflows),
+    # and base-branch updates.
+    - cron: "*/30 * * * *"
+  workflow_dispatch:
+    inputs:
+      pr:
+        description: "PR number(s), comma separated. Leave empty to sweep all candidates."
+        required: false
+        type: string
+      dry_run:
+        description: "Evaluate only; never arm or disarm auto-merge."
+        required: false
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: auto-merge-${{ github.event.pull_request.number || github.event.check_run.head_sha || github.event_name }}
+  cancel-in-progress: false
+
+jobs:
+  evaluate:
+    name: safe-auto-merge
+    if: >-
+      github.repository_owner == 'github' &&
+      (github.event_name != 'check_run' || github.event.check_run.name == 'submission-gate')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+      checks: read
+      statuses: read
+    steps:
+      - name: Checkout base branch (trusted code only)
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: "22"
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Evaluate safe auto-merge
+        env:
+          # A PAT or GitHub App token is recommended so that merges trigger
+          # downstream push workflows (merges made with GITHUB_TOKEN do not).
+          GITHUB_TOKEN: ${{ secrets.AUTO_MERGE_TOKEN || github.token }}
+          EVENT_NAME: ${{ github.event_name }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          CHECK_SHA: ${{ github.event.check_run.head_sha }}
+          INPUT_PR: ${{ inputs.pr }}
+          INPUT_DRY_RUN: ${{ inputs.dry_run }}
+        run: |
+          set -euo pipefail
+          args=(--repo "$GITHUB_REPOSITORY" --json "$RUNNER_TEMP/auto-merge.json")
+          case "$EVENT_NAME" in
+            pull_request_target)
+              args+=(--pr "$PR_NUMBER")
+              ;;
+            check_run)
+              args+=(--sha "$CHECK_SHA")
+              ;;
+            workflow_dispatch)
+              if [ -n "$INPUT_PR" ]; then args+=(--pr "$INPUT_PR"); else args+=(--all); fi
+              if [ "$INPUT_DRY_RUN" = "true" ]; then args+=(--dry-run); fi
+              ;;
+            *)
+              args+=(--all)
+              ;;
+          esac
+          node ./eng/auto-merge.mjs "${args[@]}"

.github/workflows/canvas-smoke-test-comment.yml

@@ -0,0 +1,107 @@
+name: Canvas Smoke Test — Comment
+
+# Writer half of the canvas-smoke-test reader/writer split. The reader
+# (validate-canvas-extensions.yml) runs on untrusted PR code with a read-only
+# token; this workflow only reads its artifact and never checks out PR code.
+
+on:
+  workflow_run:
+    workflows: ["Validate Canvas Extensions"]
+    types: [completed]
+
+permissions:
+  actions: read
+  issues: write
+  pull-requests: write
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: Download review artifact
+        id: download
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: canvas-smoke-test-results
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+
+      - name: Upsert PR comment
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const fs = require('fs');
+            const marker = '<!-- canvas-smoke-test -->';
+
+            if (!fs.existsSync('report.md')) {
+              core.info('No canvas smoke test report artifact found. Skipping.');
+              return;
+            }
+
+            let status = 'unknown';
+            try {
+              status = JSON.parse(fs.readFileSync('results.json', 'utf8')).status || 'unknown';
+            } catch {
+              core.warning('results.json is missing or invalid.');
+            }
+
+            const prNumber = parseInt(fs.existsSync('pr-number.txt') ? fs.readFileSync('pr-number.txt', 'utf8').trim() : '', 10)
+              || context.payload.workflow_run.pull_requests?.[0]?.number;
+            if (!prNumber) {
+              core.warning('Could not determine PR number. Skipping.');
+              return;
+            }
+
+            // The artifact is untrusted: only comment on the PR whose head matches this run.
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber,
+            });
+            if (pr.head.sha !== context.payload.workflow_run.head_sha) {
+              core.info(`PR #${prNumber} head ${pr.head.sha} does not match run head ${context.payload.workflow_run.head_sha}. Skipping stale or mismatched report.`);
+              return;
+            }
+
+            let body = fs.readFileSync('report.md', 'utf8').replace(/@/g, '@\u200b');
+            const maxLength = 65000;
+            if (body.length > maxLength) {
+              body = `${body.slice(0, maxLength)}\n\n_...(truncated)..._`;
+            }
+            if (!body.includes(marker)) {
+              body = `${marker}\n${body}`;
+            }
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              per_page: 100,
+            });
+            const existing = comments.find((comment) =>
+              comment.user?.type === 'Bot' && comment.body?.includes(marker));
+
+            if (status === 'skipped' && !existing) {
+              core.info('No canvas changes and no previous comment. Nothing to post.');
+              return;
+            }
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+              core.info(`Updated canvas smoke test comment ${existing.id}`);
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+              core.info('Created canvas smoke test comment');
+            }