♻️ The Rulekeeper - Index from one workflow that starts the Azure job

.github/workflows/rules-chat-index.yml

@@ -1,8 +1,7 @@
 name: Index Rules for The Rulekeeper
 
-# Embeds every new or changed rule into The Rulekeeper's database, from a GitHub runner. A fallback for the Container
-# Apps index job (rules-chat-reindex-trigger.yml starts that one): use it to reset an index or when the job is broken.
-# Both take the same database lock, so they never run at the same time.
+# Starts the Container Apps index job now, instead of waiting for its hourly run, and waits for it to finish. Without
+# options it embeds only the new and changed rules, like the hourly run.
 on:
   workflow_dispatch:
     inputs:
@@ -22,98 +21,106 @@ on:
         type: boolean
         default: false
 
-concurrency:
-  group: rules-chat-index-${{ inputs.environment }}
-  cancel-in-progress: false
-
 permissions:
   id-token: write
   contents: read
 
 jobs:
-  index:
-    name: Index Rules
+  start-index-job:
+    name: Start Index Job
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
-    timeout-minutes: 120
+    # Covers the index job's own two-hour limit (replicaTimeout in infra/modules/rulesChatIndexJob.bicep).
+    timeout-minutes: 130
+    defaults:
+      run:
+        # Naming the shell adds -o pipefail, so a failed az call in a pipeline fails the step.
+        shell: bash
     env:
       # Resource names follow infra/deploy.ps1, which names the production resources "prod".
       RESOURCE_SUFFIX: ${{ inputs.environment == 'production' && 'prod' || 'staging' }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: .nvmrc
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.10.0
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
       - name: Azure CLI - Login
         uses: azure/login@v2
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
 
-      # Indexes the branch the Container Apps job reads, so the two never overwrite each other with different content.
-      # There is no default branch: indexing one the job doesn't read would put the wrong content in the shared index
-      # until the job's next run changes it back.
-      - name: Read the index job's content branch
-        id: content_branch
+      # The index script takes a database lock, so a run started while another is going waits up to 30 minutes for it to
+      # finish. If it gets the lock it indexes as usual; if not, it indexes nothing and still ends Succeeded.
+      - name: Start the index job
+        id: start
         env:
           RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
+          REEMBED: ${{ inputs.reembed }}
+          ALLOW_REMOVALS: ${{ inputs.allow_removals }}
         run: |
           JOB_NAME="caj-sswrules-chat-index-$RESOURCE_SUFFIX"
-          if ! BRANCH=$(az containerapp job show --resource-group "$RESOURCE_GROUP" --name "$JOB_NAME" \
-            --query "properties.template.containers[0].env[?name=='RULES_CONTENT_BRANCH'].value | [0]" --output tsv); then
-            echo "::error::Couldn't read $JOB_NAME. Deploy the environment, or check the pipeline's access to the job."
-            exit 1
+          ARGS=()
+          if [ "$REEMBED" = "true" ]; then ARGS+=(--reembed); fi
+          if [ "$ALLOW_REMOVALS" = "true" ]; then ARGS+=(--allow-removals); fi
+
+          if [ ${#ARGS[@]} -eq 0 ]; then
+            EXECUTION=$(az containerapp job start --name "$JOB_NAME" --resource-group "$RESOURCE_GROUP" --query name --output tsv)
+          else
+            # az containerapp job start --args sends a container with only the arguments, not the job's image and settings.
+            # So this starts the run with the job's own container, plus the arguments.
+            JOB_ID=$(az containerapp job show --name "$JOB_NAME" --resource-group "$RESOURCE_GROUP" --query id --output tsv)
+            ARGS_JSON=$(printf '%s\n' "${ARGS[@]}" | jq --raw-input . | jq --slurp --compact-output .)
+            TEMPLATE=$(az containerapp job show --name "$JOB_NAME" --resource-group "$RESOURCE_GROUP" \
+              --query "properties.template.{containers: containers, initContainers: initContainers}" --output json \
+              | jq --argjson args "$ARGS_JSON" '.containers[0].args = $args')
+            EXECUTION=$(az rest --method post --url "https://management.azure.com$JOB_ID/start?api-version=2024-03-01" \
+              --headers "Content-Type=application/json" --body "$TEMPLATE" --query name --output tsv)
           fi
-          if [ -z "$BRANCH" ]; then
-            echo "::error::$JOB_NAME has no RULES_CONTENT_BRANCH setting. Deploy the environment to set it."
+          if [ -z "$EXECUTION" ]; then
+            echo "::error::Azure didn't return the name of the started execution."
             exit 1
           fi
-          echo "Indexing SSW.Rules.Content $BRANCH"
-          echo "branch=$BRANCH" >> "$GITHUB_OUTPUT"
-
-      - name: Checkout SSW.Rules.Content
-        uses: actions/checkout@v4
-        with:
-          repository: SSWConsulting/SSW.Rules.Content
-          ref: ${{ steps.content_branch.outputs.branch }}
-          path: content
-          sparse-checkout: public/uploads/rules
+          echo "Started execution $EXECUTION${ARGS[*]:+ with ${ARGS[*]}}. Its logs are in the job's Execution history in the Azure portal."
+          echo "execution=$EXECUTION" >> "$GITHUB_OUTPUT"
 
-      - name: Index rules
+      - name: Wait for the index job
         env:
           RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
-          # Resolved from the scripts folder, so this is ./content.
-          LOCAL_CONTENT_RELATIVE_PATH: ../content
-          RULES_CHAT_EMBEDDING_MODEL: text-embedding-3-large
-          RULES_CHAT_EMBEDDING_DIMENSIONS: "1024"
-          RULES_CHAT_SQL_DATABASE: RulesChat
+          EXECUTION: ${{ steps.start.outputs.execution }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
         run: |
-          SQL_SERVER_NAME="sql-sswrules-chat-$RESOURCE_SUFFIX"
-          export RULES_CHAT_SQL_SERVER="$SQL_SERVER_NAME.database.windows.net"
-          export RULES_CHAT_AI_BASE_URL="https://aif-sswrules-chat-$RESOURCE_SUFFIX.openai.azure.com/openai/v1"
+          JOB_NAME="caj-sswrules-chat-index-$RESOURCE_SUFFIX"
+
+          # The Azure CLI can't renew its token with the GitHub token it logged in with, because that token expires after a
+          # few minutes (AADSTS700024). So a long run logs in again with a new one.
+          login_again() {
+            local token
+            token=$(curl -fsS -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
+              "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=api://AzureADTokenExchange" | jq -r .value)
+            az login --service-principal --username "$AZURE_CLIENT_ID" --tenant "$AZURE_TENANT_ID" --federated-token "$token" --output none
+            az account set --subscription "$AZURE_SUBSCRIPTION_ID"
+          }
 
-          # The SQL firewall only admits Azure services. Open it to this runner for the length of this step.
-          RULE="github-actions-${{ github.run_id }}-${{ github.run_attempt }}"
-          RUNNER_IP=$(curl -fsS https://api.ipify.org)
-          az sql server firewall-rule create --resource-group "$RESOURCE_GROUP" --server "$SQL_SERVER_NAME" --name "$RULE" \
-            --start-ip-address "$RUNNER_IP" --end-ip-address "$RUNNER_IP" --output none
-          trap 'az sql server firewall-rule delete --resource-group "$RESOURCE_GROUP" --server "$SQL_SERVER_NAME" --name "$RULE" --output none' EXIT
+          status() {
+            az containerapp job execution show --name "$JOB_NAME" --resource-group "$RESOURCE_GROUP" \
+              --job-execution-name "$EXECUTION" --query properties.status --output tsv
+          }
 
-          # Signs in to SQL and Foundry as the pipeline, through the Azure CLI login above.
-          ARGS=(--refresh-content)
-          if [ "${{ inputs.reembed }}" = "true" ]; then ARGS+=(--reembed); fi
-          if [ "${{ inputs.allow_removals }}" = "true" ]; then ARGS+=(--allow-removals); fi
-          pnpm rules-chat:index "${ARGS[@]}"
+          while :; do
+            if ! STATUS=$(status); then
+              echo "Reading the execution's status failed. Logging in to Azure again."
+              login_again
+              STATUS=$(status)
+            fi
+            case "$STATUS" in
+              Succeeded)
+                echo "Execution $EXECUTION succeeded."
+                break
+                ;;
+              Failed | Stopped | Degraded)
+                echo "::error::Execution $EXECUTION ended $STATUS. Its logs are in the job's Execution history in the Azure portal."
+                exit 1
+                ;;
+            esac
+            sleep 60
+          done

infra/README.md

@@ -56,8 +56,7 @@ Only the workflow builds that image. A manual `deploy.ps1` run keeps the image t
 It runs:
 
 - **Every hour**, on the hour. A run with no changed rules only clones the rules and compares hashes, so a new or edited rule reaches the chat within an hour.
-- **By hand**, with the **Re-index Rules for The Rulekeeper** workflow or the job's **Run now** in the Azure portal.
-- **By hand from a GitHub runner** with **Index Rules for The Rulekeeper**, the fallback for resetting an index.
+- **By hand**, with the **Index Rules for The Rulekeeper** workflow or the job's **Run now** in the Azure portal. The workflow can also re-embed every rule, or let a run remove more than a fifth of the index. It waits for the run and fails if the run fails.
 
 Runs never overlap: the index script takes a database lock. A second run waits up to 30 minutes for it, fetches the content branch again, and only then reads the rules, so it never indexes older content than the first run did. A run still waiting after 30 minutes, such as during a first full index or a re-embed, indexes nothing and isn't counted as a failure. The script also stops if the tables don't exist, which means the environment hasn't been deployed yet.
 
@@ -68,7 +67,7 @@ Runs never overlap: the index script takes a database lock. A second run waits u
 
 Without it, no alerts are created. Both alerts search the job's logs for markers the index script prints (`RULES_CHAT_INDEX_FAILED` and `RULES_CHAT_INDEX_SUCCEEDED`). The staleness alert also fires in the two days after a new environment's first deploy, until the job first succeeds.
 
-The job reads the content branch the site shows, resolved the way the site build resolves it: a manual deploy's `tina_branch`, else the `NEXT_PUBLIC_TINA_BRANCH` GitHub variable, else `main`. If SSW.Rules.Content has no such branch, the job reads `main`. The GitHub runner fallback reads the same branch as the job, and fails if it can't read the job's branch.
+The job reads the content branch the site shows, resolved the way the site build resolves it: a manual deploy's `tina_branch`, else the `NEXT_PUBLIC_TINA_BRANCH` GitHub variable, else `main`. If SSW.Rules.Content has no such branch, the job reads `main`.
 
 ### Looking at the data
 

infra/main.bicep

@@ -256,7 +256,6 @@ module rulesChatFoundryModule 'modules/rulesChatFoundry.bicep' = if (deploysShar
       rulesChatIdentity.properties.principalId
       rulesChatIndexJobIdentity!.properties.principalId
     ]
-    pipelinePrincipalId: servicePrincipalObjectId
     tags: tags
   }
 }

infra/modules/rulesChatFoundry.bicep

@@ -19,9 +19,6 @@ param embeddingModel object
 @description('Principal IDs of the managed identities that call the models')
 param callerPrincipalIds array
 
-@description('Optional: object ID of the deployment pipeline\'s service principal, which embeds the rules when indexing')
-param pipelinePrincipalId string = ''
-
 @description('Tags to apply to the resource')
 param tags object = {}
 
@@ -90,16 +87,6 @@ resource embeddingDeployment 'Microsoft.CognitiveServices/accounts/deployments@2
   ]
 }
 
-resource pipelineRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(pipelinePrincipalId)) {
-  name: guid(foundry.id, pipelinePrincipalId, cognitiveServicesOpenAIUserRoleId)
-  scope: foundry
-  properties: {
-    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', cognitiveServicesOpenAIUserRoleId)
-    principalId: pipelinePrincipalId
-    principalType: 'ServicePrincipal'
-  }
-}
-
 resource callerRoles 'Microsoft.Authorization/roleAssignments@2022-04-01' = [
   for callerPrincipalId in callerPrincipalIds: {
     name: guid(foundry.id, callerPrincipalId, cognitiveServicesOpenAIUserRoleId)

scripts/rules-chat/index-rules.mjs

@@ -7,7 +7,7 @@
 //   pnpm rules-chat:index --sample <n>   index only the first n rules, for a quick local setup
 //   pnpm rules-chat:index --show <uri>   print the chunks for one rule without indexing
 //   pnpm rules-chat:index --refresh-content   fetch the content checkout's branch again once the lock is held, discarding
-//                                             local changes in it (for the job and the runner workflow, not a local checkout)
+//                                             local changes in it (for the job, not a local checkout)
 
 import { execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
@@ -175,10 +175,10 @@ async function main() {
     );
   }
 
-  // One connection holds the lock for the whole run, so the hourly job, a run started by hand and the GitHub runner workflow
-  // never index the same database at once. The lock ends when the connection closes, so the pool keeps it open even
-  // while idle (min: 1). A run that finds the lock taken waits for it. Its checkout may be older than the one the
-  // running job indexed, so the rules are read only once the lock is held, after --refresh-content fetches them again.
+  // One connection holds the lock for the whole run, so the hourly job and a run started by hand never index the same
+  // database at once. The lock ends when the connection closes, so the pool keeps it open even while idle (min: 1). A
+  // run that finds the lock taken waits for it. Its checkout may be older than the one the running job indexed, so the
+  // rules are read only once the lock is held, after --refresh-content fetches them again.
   let lockConnection;
   let pool;
   try {

scripts/rules-chat/lock.mjs

@@ -1,5 +1,5 @@
-// The application lock that keeps the hourly job, a run started by hand and the GitHub runner workflow from indexing the
-// same database at once. index-rules.mjs takes it with sp_getapplock on a connection of its own.
+// The application lock that keeps the hourly job and a run started by hand from indexing the same database at once.
+// index-rules.mjs takes it with sp_getapplock on a connection of its own.
 
 export const INDEX_LOCK = "rules-chat-index";