Monitor enroll post unlock

.gitignore

@@ -26,5 +26,8 @@ helm/wind-river-cloud-platform-deployment-manager/templates/rbac
 .vscode
 .kiro
 
+# log file
+*.log
+
 # Controller-gen generated empty CRD file
 config/crd/bases/_.yaml

DEBUGGING.md

@@ -82,7 +82,7 @@ The "Args:" section would look like the following if the log level was changed f
 the default 'info' to a new value (2). Log level should be one of 'debug', 'info',
 'error', or any integer value > 0 which corresponds to custom debug levels of
 increasing verbosity. Stack trace level can be also modified with the
-"--zap-stack trace-level" parameter. The following shows to change stack trace level
+"--zap-stacktrace-level" parameter. The following shows to change stack trace level
 from the default 'panic' to 'info'. Stack trace level should be one of 'info',
 'error', 'panic'.
 

Dockerfile

@@ -25,6 +25,7 @@ RUN go mod download
 COPY cmd/ cmd/
 COPY api/ api/
 COPY common/ common/
+COPY logger/ logger/
 COPY platform/ platform/
 COPY internal/ internal/
 COPY scripts/ scripts/

common/apierrors.go

@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/* Copyright(c) 2026 Wind River Systems, Inc. */
+
+package common
+
+import (
+	"errors"
+	"strings"
+
+	perrors "github.com/pkg/errors"
+)
+
+// apiResponseError is the subset of the gophercloud StarlingX FaultError that
+// carries the details of a rejected (non 2xx) API response.  FaultError is
+// installed as the StarlingX client's default error context, so every rejected
+// StarlingX request is returned as one, regardless of the status code.  The
+// interface is matched with errors.As so that errors annotated with
+// perrors.Wrap (which implements Unwrap) are also recognized.
+type apiResponseError interface {
+	error
+	GetStatusCode() int
+	GetBody() []byte
+}
+
+// extractAPIResponseError returns the gophercloud response error carried by
+// err, or false if err was not produced by a rejected (non 2xx) StarlingX API
+// response.
+func extractAPIResponseError(err error) (apiResponseError, bool) {
+	var respErr apiResponseError
+	if errors.As(err, &respErr) {
+		return respErr, true
+	}
+	return nil, false
+}
+
+// IsAPIResponseError returns true if the error (or the error it wraps) was
+// produced because the system API rejected a request with a non 2xx status.
+func IsAPIResponseError(err error) bool {
+	_, ok := extractAPIResponseError(err)
+	return ok
+}
+
+// APIErrorStatusCode returns the HTTP status code associated to a rejected
+// system API request, or 0 if the error is not an API response error.
+func APIErrorStatusCode(err error) int {
+	if resp, ok := extractAPIResponseError(err); ok {
+		return resp.GetStatusCode()
+	}
+	return 0
+}
+
+// APIErrorReason returns a concise, human readable explanation of why an
+// operation failed.  For rejected system API requests the reason reported by
+// the server (i.e., the sysinv faultstring) is returned: the gophercloud
+// FaultError already extracts it from the response body and strips any remote
+// Python traceback, so its Error() string is the reason.  For any other error
+// the error string is returned.
+func APIErrorReason(err error) string {
+	if err == nil {
+		return ""
+	}
+	if resp, ok := extractAPIResponseError(err); ok {
+		return resp.Error()
+	}
+	return err.Error()
+}
+
+// APIErrorOperation returns the annotations added on top of the root cause of
+// an error with perrors.Wrap (e.g., "failed to create route ...").  These
+// describe the operation that was being attempted.  An empty string is
+// returned if the error was not annotated.
+func APIErrorOperation(err error) string {
+	// Note: gophercloud errors embed the response body ([]byte) so they are
+	// not comparable; check for a wrapper instead of comparing err to its
+	// cause, which would panic.
+	if _, wrapped := err.(interface{ Cause() error }); !wrapped {
+		return ""
+	}
+	cause := perrors.Cause(err)
+	if cause == nil {
+		return ""
+	}
+	return strings.TrimSuffix(err.Error(), ": "+cause.Error())
+}

common/apierrors_test.go

@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/* Copyright(c) 2026 Wind River Systems, Inc. */
+
+package common
+
+import (
+	"errors"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/starlingx"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	perrors "github.com/pkg/errors"
+)
+
+var _ = Describe("API error utilities", func() {
+	body := []byte(`{"error_message": "{\"debuginfo\": null, \"faultcode\": \"Client\", \"faultstring\": \"Rejected: host not ready\"}"}`)
+
+	// fault builds the StarlingX FaultError that gophercloud returns for a
+	// rejected (non 2xx) StarlingX request, regardless of the status code.
+	fault := func(code int) error {
+		return starlingx.FaultError{
+			ErrUnexpectedResponseCode: gophercloud.ErrUnexpectedResponseCode{
+				Method: "PATCH", URL: "http://sysinv/v1/ihosts/1", Actual: code, Body: body,
+			},
+		}
+	}
+
+	DescribeTable("should extract the response from a rejected request, wrapped or not",
+		func(in error, code int) {
+			Expect(IsAPIResponseError(in)).To(BeTrue())
+			Expect(APIErrorStatusCode(in)).To(Equal(code))
+			Expect(APIErrorReason(in)).To(Equal("Rejected: host not ready"))
+		},
+		Entry("400", fault(400), 400),
+		Entry("403", fault(403), 403),
+		Entry("404", fault(404), 404),
+		Entry("405", fault(405), 405),
+		Entry("408", fault(408), 408),
+		Entry("409", fault(409), 409),
+		Entry("422", fault(422), 422),
+		Entry("429", fault(429), 429),
+		Entry("500", fault(500), 500),
+		Entry("503", fault(503), 503),
+		Entry("wrapped 409",
+			perrors.Wrap(fault(409), "failed to create route"), 409),
+	)
+
+	It("should report the wrap annotation as the operation", func() {
+		in := perrors.Wrap(fault(409), "failed to create route")
+		Expect(APIErrorOperation(in)).To(Equal("failed to create route"))
+	})
+
+	It("should report no operation for an unwrapped response error", func() {
+		Expect(APIErrorOperation(fault(400))).To(Equal(""))
+	})
+
+	DescribeTable("should not treat other errors as response errors",
+		func(in error) {
+			Expect(IsAPIResponseError(in)).To(BeFalse())
+			Expect(APIErrorStatusCode(in)).To(Equal(0))
+		},
+		Entry("nil", nil),
+		Entry("plain error", errors.New("boom")),
+		Entry("gophercloud error without a response", gophercloud.ErrTimeOut{}),
+		Entry("re-authentication failure", &gophercloud.ErrUnableToReauthenticate{}),
+	)
+
+	It("should return the plain error string as the reason for a non-response error", func() {
+		Expect(APIErrorReason(errors.New("boom"))).To(Equal("boom"))
+		Expect(APIErrorReason(nil)).To(Equal(""))
+	})
+})

docs/charts/index.yaml

@@ -3,12 +3,12 @@ entries:
   wind-river-cloud-platform-deployment-manager:
   - apiVersion: v1
     appVersion: "1.0"
-    created: "2026-09-24T15:55:44.767871556-03:00"
+    created: "2026-10-30T13:28:33.21273636-04:00"
     description: A Helm chart for the Deployment Manager
-    digest: 3b60472403dad33b2175762083c7b7500e82f5697e806bc796a63878c1fa777c
+    digest: 81e23edcd41bea256997eb13469584bfe97d5adb3c37f03458c04729583a264a
     icon: https://www.windriver.com/resources/images/wr-logo-red-2018.png
     name: wind-river-cloud-platform-deployment-manager
     urls:
     - wind-river-cloud-platform-deployment-manager-26.10.0.tgz
     version: 26.10.0
-generated: "2026-09-24T15:55:44.76698159-03:00"
+generated: "2026-10-30T13:28:33.211476505-04:00"

docs/playbooks/roles/common/tasks/apply-config.yaml → docs/playbooks/roles/common/apply-config/tasks/main.yaml

@@ -1,6 +1,6 @@
 ---
 # SPDX-License-Identifier: Apache-2.0
-# Copyright(c) 2025 Wind River Systems, Inc.
+# Copyright(c) 2025-2026 Wind River Systems, Inc.
 - name: Replace OS_REGION_NAME when target is a subcloud
   replace:
     path: "{{ deploy_config }}"

docs/playbooks/roles/initial-config/tasks/get-administrative-state.yaml → docs/playbooks/roles/common/get-administrative-state/tasks/main.yaml

@@ -1,6 +1,6 @@
 ---
 # SPDX-License-Identifier: Apache-2.0
-# Copyright(c) 2025 Wind River Systems, Inc.
+# Copyright(c) 2025-2026 Wind River Systems, Inc.
 - name: Search administrativeState into host resource
   shell: >-
     kubectl get host {{ hostname }}

docs/playbooks/roles/common/lock-unlock/defaults/main.yaml

@@ -0,0 +1,20 @@
+---
+# SPDX-License-Identifier: Apache-2.0
+# Copyright(c) 2026 Wind River Systems, Inc.
+
+# How long to wait for the unlock reboot triggered by the deployment manager.
+dm_unlock_timeout: 1200
+
+# How long the deployment configuration must have been applied before the
+# unlock rejections are reported: sysinv rejecting the unlock while puppet is
+# still applying the configuration is expected for the first few minutes.
+dm_unlock_monitor_delay: 180
+
+# How often the unlock progress is checked while waiting for the reboot.
+dm_unlock_monitor_interval: 20
+
+# Where the unlock diagnostics are collected. Defaults to a temporary file in
+# the home directory of the ansible user, see tasks/prepare.yaml. Override it
+# with an absolute path on a persistent filesystem that the ansible user can
+# write: no task of this role uses become, and /tmp is wiped by the reboot.
+# dm_unlock_monitor_log: /home/sysadmin/dm-unlock-monitor.log

docs/playbooks/roles/common/lock-unlock/files/dm-unlock-monitor.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# Copyright(c) 2026 Wind River Systems, Inc.
+#
+# Collects the reasons why a host has not been unlocked yet.
+#
+# The deployment manager unlocks the host once the configuration has been
+# applied, and sysinv may reject that unlock while puppet is still applying
+# parts of the configuration (SR-IOV interfaces, host cpus, ...). The
+# deployment manager retries, so those rejections are only visible in its pod
+# log.
+#
+# This script is started detached by roles/common/lock-unlock before the
+# playbook starts waiting for the unlock reboot, because that wait blocks and
+# cannot report anything while it runs. The playbook reports what was
+# collected here once the wait is over. Only the pod log is read, so the
+# collector never interferes with the unlock itself.
+#
+# The collector is killed by the unlock reboot, by the playbook, or by the
+# ansible async timeout, whichever comes first.
+#
+# Usage:
+#   dm-unlock-monitor.sh <hostname> <pod-namespace> <delay> <interval> <logfile>
+#
+# No rejection is reported during the first <delay> seconds: sysinv rejecting
+# the unlock while the configuration is being applied is expected. The pod log
+# is then checked every <interval> seconds, and a line is appended to
+# <logfile> whenever the reported status changes.
+
+set -u
+
+hostname=${1}
+pod_namespace=${2}
+delay=${3}
+interval=${4}
+logfile=${5}
+
+export KUBECONFIG=/etc/kubernetes/admin.conf
+
+readonly DEPLOYMENT="deployments/platform-deployment-manager"
+readonly CONTAINER="manager"
+
+# Logged by the deployment manager for every unlock request it sends.
+readonly ATTEMPT_PATTERN="unlocking host"
+
+# A rejected unlock is logged either as
+#   request rejected by system API {..., "operation": "unlock host <name>",
+#   "status": 400, "reason": "<reason>"}
+# or, by a deployment manager built before commit 07fd868c, as
+#   Failed to unlock <name>: <reason>
+# Both are matched: the build running on the target is not known here, and the
+# pod log may hold neither (no rejection logged, or the log was rotated).
+readonly REJECTION_PATTERN="unlock host ${hostname}\""
+readonly LEGACY_REJECTION_PATTERN="Failed to unlock ${hostname}: "
+
+# Start fresh: a log left behind by a previous (failed) run must not be
+# reported as the result of this one.
+: >"${logfile}" || exit 1
+
+# Used by the playbook to stop the collector. Matching the command line
+# instead (pkill -f) would also match any other process that happens to
+# mention this script, including the playbook invoking it.
+pidfile="${logfile}.pid"
+echo $$ >"${pidfile}"
+trap 'rm -f "${pidfile}"' EXIT
+
+log() {
+    printf '%s  %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${1}" >>"${logfile}"
+}
+
+# Prints the reason reported by sysinv for each rejected unlock of this host
+# found in "${logs}", in the order they were logged, one per line.
+rejections() {
+    {
+        grep -F "${REJECTION_PATTERN}" <<<"${logs}" |
+            sed -nE 's/.*"reason": ?"(.*)"\}[[:space:]]*$/\1/p'
+        grep -F "${LEGACY_REJECTION_PATTERN}" <<<"${logs}" |
+            sed -E "s/.*${LEGACY_REJECTION_PATTERN}//"
+    } |
+        # The reasons are embedded in a JSON object, and a reason reported by
+        # sysinv can itself quote a value.
+        sed -e 's/\\"/"/g' -e 's/[[:space:]]*$//' |
+        grep -v '^$'
+}
+
+start=$(date +%s)
+previous=""
+
+# Nothing is written until there is something to report, so that a host that
+# is unlocked right away leaves an empty log and nothing to show.
+while true; do
+    sleep "${interval}"
+
+    elapsed=$(($(date +%s) - start))
+    if [ "${elapsed}" -lt "${delay}" ]; then
+        continue
+    fi
+
+    # Read a bit more than the monitored window so that the first report also
+    # covers the unlock attempts made before the collector was started.
+    if ! logs=$(kubectl -n "${pod_namespace}" logs "${DEPLOYMENT}" "${CONTAINER}" \
+        --since="$((elapsed + 60))s" 2>&1); then
+        # Expected while the unlock drains the pods, and whenever the
+        # kube-apiserver is unreachable. Keep polling.
+        status="the deployment manager pod log is not available (the pods may"
+        status="${status} have been drained by the unlock): $(head -1 <<<"${logs}")"
+    else
+        attempts=$(grep -c -F "${ATTEMPT_PATTERN}" <<<"${logs}")
+        reasons=$(rejections)
+
+        if [ -n "${reasons}" ]; then
+            status="unlock of ${hostname} rejected $(grep -c . <<<"${reasons}")"
+            status="${status} time(s) out of ${attempts} attempt(s), reason(s)"
+            status="${status} reported by sysinv:"
+            # Distinct reasons only: the same one is reported on every retry.
+            while read -r reason; do
+                status="${status}"$'\n'"    ${reason}"
+            done <<<"$(awk '!seen[$0]++' <<<"${reasons}")"
+        else
+            status="${hostname} is not unlocked yet and the deployment manager"
+            status="${status} logged no unlock rejection (${attempts} unlock"
+            status="${status} attempt(s) so far)"
+            # Without a rejection to report, the most recent errors logged by
+            # the deployment manager are the only hint available.
+            errors=$(grep -E '(^|[[:space:]])ERROR([[:space:]]|$)' <<<"${logs}" |
+                awk '!seen[$0]++' | tail -5)
+            if [ -n "${errors}" ]; then
+                status="${status}, last error(s) logged:"
+                while read -r line; do
+                    status="${status}"$'\n'"    ${line}"
+                done <<<"${errors}"
+            fi
+        fi
+    fi
+
+    # Only report changes: the status is otherwise repeated every interval
+    # until the host is unlocked.
+    if [ "${status}" != "${previous}" ]; then
+        log "after ${elapsed}s: ${status}"
+        previous="${status}"
+    fi
+done

docs/playbooks/roles/initial-config/files/shutdown → docs/playbooks/roles/common/lock-unlock/files/shutdown

@@ -1,6 +1,6 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
-# Copyright(c) 2025 Wind River Systems, Inc.
+# Copyright(c) 2025-2026 Wind River Systems, Inc.
 
 # No-op shutdown script for the ansible reboot task. The reboot task calls shutdown,
 # which we override using search_path to invoke this fake script. The task then waits