fix(goose): read token totals from accumulated_usage on goose 1.39+

agents/entire-agent-goose/AGENT.md

@@ -94,6 +94,7 @@ real captured payloads unless marked `(unverified)`.
 - User prompt field: `conversation[].content[].text` where `role=="user"` and `content[].type=="text"` (skip `toolResponse` user messages).
 - Modified files field: `conversation[].content[]` where `type=="toolRequest"`, `toolCall.value.name` in `write`/`text_editor`/`edit`-style tools; file path at `toolCall.value.arguments.path`. Shell commands at `arguments.command` for heuristics.
 - Token usage field: session-level `accumulated_input_tokens`/`accumulated_output_tokens` (export and `sessions` table). Per-message `tokens` column exists but was NULL in verification. No cache token split available.
+- Token usage since goose 1.39.0 **(read from goose source, not re-run)**: the flat token fields are replaced by nested `usage` (current context window) and `accumulated_usage` (whole session) objects with nullable `input_tokens`, `output_tokens`, `total_tokens`, `cache_read_input_tokens`, `cache_write_input_tokens`. goose's `input_tokens` includes the cache reads and writes, so `calculate-tokens` subtracts them to report fresh input. The export spells them `cache_*_input_tokens`; the `sessions` table columns are `cache_read_tokens`/`cache_write_tokens`.
 - Example tool entry **(verified)**:
   ```json
   {"type":"toolRequest","id":"toolu_...","toolCall":{"status":"success",
@@ -129,7 +130,7 @@ real captured payloads unless marked `(unverified)`.
 | `extract-prompts` | user text messages | parse `conversation` | transcript_analyzer |
 | `extract-summary` | session `name` | auto-generated session name (`"File verification request"`) as summary | transcript_analyzer |
 | `prepare-transcript` | `goose session export` | shell out: `goose session export --session-id <id> --format json -o <scratch>`, then materialize as JSONL and write atomically to `session_ref` | transcript_preparer |
-| `calculate-tokens` | export token fields | `accumulated_input/output_tokens`; `api_call_count` = assistant message count | token_calculator |
+| `calculate-tokens` | export token fields | `accumulated_usage`, else `usage`, else flat `accumulated_input/output_tokens`; `api_call_count` = assistant message count | token_calculator |
 
 ## Selected Capabilities
 | Capability | Declared | Justification |
@@ -138,7 +139,7 @@ real captured payloads unless marked `(unverified)`.
 | transcript_analyzer | true | export JSON has prompts, tool calls with file paths |
 | transcript_preparer | true | transcript must be materialized from sessions.db via `goose session export` |
 | compact_transcript | true | converts the export to Entire Transcript Format so assistant responses and tool calls render in `entire explain` / web UI |
-| token_calculator | true | accumulated input/output tokens in export |
+| token_calculator | true | accumulated token totals in export (with cache split on goose 1.39.0+) |
 | text_generator | false | possible later via headless `goose run -t --no-session`, but output includes UI text; defer |
 | hook_response_writer | false | no clean native channel for display-only messages (Stop deny blocks the turn) |
 | subagent_aware_extractor | false | subagent sessions are separate `sessions.db` rows (`session_type='subagent'`), not per-dir transcripts; defer |

agents/entire-agent-goose/README.md

@@ -20,8 +20,10 @@ goose 1.37.0).
   (`~/.local/share/goose/sessions/sessions.db`). There is no per-session
   file, so `prepare-transcript` materializes one via
   `goose session export --session-id <id> --format json`.
-- **Tokens** — accumulated input/output token totals come from the exported
-  session JSON.
+- **Tokens** — session token totals come from the exported session JSON:
+  `accumulated_usage` (falling back to `usage`) on goose 1.39.0+, with its
+  cache read/write split, or the flat `accumulated_input/output_tokens` fields
+  on older goose.
 
 ## Build
 

agents/entire-agent-goose/internal/goose/session_jsonl.go

@@ -69,10 +69,23 @@ type gooseSessionMeta struct {
 	CreatedAt              string            `json:"created_at,omitempty"`
 	AccumulatedInputTokens int               `json:"accumulated_input_tokens,omitempty"`
 	AccumulatedOutput      int               `json:"accumulated_output_tokens,omitempty"`
+	Usage                  *gooseUsage       `json:"usage,omitempty"`
+	AccumulatedUsage       *gooseUsage       `json:"accumulated_usage,omitempty"`
 	ProviderName           string            `json:"provider_name,omitempty"`
 	ModelConfig            *gooseModelConfig `json:"model_config,omitempty"`
 }
 
+// gooseUsage is goose's Usage object, which replaced the flat
+// accumulated_*_tokens export fields in goose 1.39.0. Every field is nullable
+// in goose (Option<i32>), so nil means "not reported" rather than zero.
+type gooseUsage struct {
+	InputTokens           *int `json:"input_tokens,omitempty"`
+	OutputTokens          *int `json:"output_tokens,omitempty"`
+	TotalTokens           *int `json:"total_tokens,omitempty"`
+	CacheReadInputTokens  *int `json:"cache_read_input_tokens,omitempty"`
+	CacheWriteInputTokens *int `json:"cache_write_input_tokens,omitempty"`
+}
+
 type gooseModelConfig struct {
 	ModelName string `json:"model_name,omitempty"`
 }

agents/entire-agent-goose/internal/goose/tokens_test.go

@@ -0,0 +1,171 @@
+package goose
+
+import (
+	"testing"
+
+	"github.com/entireio/external-agents/agents/entire-agent-goose/internal/protocol"
+)
+
+// nestedUsageExport is a trimmed `goose session export --format json` document
+// in the shape goose has written since 1.39.0, where the flat
+// accumulated_*_tokens fields were replaced by nested Usage objects. The
+// accumulated_usage figures are the ones reported against goose 1.46.0 in
+// upstream issue 74; usage (the current context window) is deliberately smaller
+// so the tests can tell which object was read.
+//
+// goose documents Usage.input_tokens as all prompt tokens, cache reads and
+// writes included (crates/goose-provider-types/src/conversation/token_usage.rs),
+// so fresh input here is 124611 - 112820 - 11777 = 14, and
+// 14 + 112820 + 11777 + 394 = 125005 = total_tokens.
+const nestedUsageExport = `{
+  "id": "20260819_1",
+  "working_dir": "/tmp/workspace",
+  "name": "Nested usage session",
+  "created_at": "2026-08-19T10:00:00Z",
+  "updated_at": "2026-08-19T10:05:00Z",
+  "usage": {
+    "input_tokens": 60000,
+    "output_tokens": 100,
+    "total_tokens": 60100,
+    "cache_read_input_tokens": 50000,
+    "cache_write_input_tokens": 9000
+  },
+  "accumulated_usage": {
+    "input_tokens": 124611,
+    "output_tokens": 394,
+    "total_tokens": 125005,
+    "cache_read_input_tokens": 112820,
+    "cache_write_input_tokens": 11777
+  },
+  "accumulated_cost": null,
+  "message_count": 2,
+  "provider_name": "anthropic",
+  "model_config": {"model_name": "claude-sonnet-4-5"},
+  "conversation": [
+    {"id": "m1", "role": "user", "created": 1787133600, "content": [{"type": "text", "text": "hi"}]},
+    {"id": "m2", "role": "assistant", "created": 1787133601, "content": [{"type": "text", "text": "hello"}]}
+  ]
+}`
+
+const nestedConversation = `"conversation": [
+    {"id": "m1", "role": "user", "created": 1787133600, "content": [{"type": "text", "text": "hi"}]},
+    {"id": "m2", "role": "assistant", "created": 1787133601, "content": [{"type": "text", "text": "hello"}]}
+  ]`
+
+func tokenExport(usageFields string) string {
+	return `{"id": "20260819_1", "name": "s", ` + usageFields + `, ` + nestedConversation + `}`
+}
+
+func TestCalculateTokensExportShapes(t *testing.T) {
+	tests := []struct {
+		name   string
+		export string
+		want   protocol.TokenUsageResponse
+	}{
+		{
+			name:   "accumulated_usage is preferred over usage",
+			export: nestedUsageExport,
+			want: protocol.TokenUsageResponse{
+				InputTokens: 14, OutputTokens: 394,
+				CacheReadTokens: 112820, CacheCreationTokens: 11777,
+				APICallCount: 1,
+			},
+		},
+		{
+			name: "usage is used when accumulated_usage is all null",
+			export: tokenExport(`
+  "usage": {"input_tokens": 1500, "output_tokens": 40, "total_tokens": 1540, "cache_read_input_tokens": 1000, "cache_write_input_tokens": 200},
+  "accumulated_usage": {"input_tokens": null, "output_tokens": null, "total_tokens": null, "cache_read_input_tokens": null, "cache_write_input_tokens": null}`),
+			want: protocol.TokenUsageResponse{
+				InputTokens: 300, OutputTokens: 40,
+				CacheReadTokens: 1000, CacheCreationTokens: 200,
+				APICallCount: 1,
+			},
+		},
+		{
+			name:   "usage is used when accumulated_usage is absent",
+			export: tokenExport(`"usage": {"input_tokens": 70, "output_tokens": 7}`),
+			want:   protocol.TokenUsageResponse{InputTokens: 70, OutputTokens: 7, APICallCount: 1},
+		},
+		{
+			name:   "pre-1.39 flat accumulated fields still read",
+			export: exportFixture,
+			want:   protocol.TokenUsageResponse{InputTokens: 5650, OutputTokens: 124, APICallCount: 2},
+		},
+		{
+			name: "accumulated_usage without input or output counts falls back to usage",
+			export: tokenExport(`
+  "usage": {"input_tokens": 300, "output_tokens": 200, "total_tokens": 500, "cache_read_input_tokens": null, "cache_write_input_tokens": null},
+  "accumulated_usage": {"input_tokens": null, "output_tokens": null, "total_tokens": 500, "cache_read_input_tokens": null, "cache_write_input_tokens": null}`),
+			want: protocol.TokenUsageResponse{InputTokens: 300, OutputTokens: 200, APICallCount: 1},
+		},
+		{
+			name: "null cache fields count as no cache",
+			export: tokenExport(`
+  "accumulated_usage": {"input_tokens": 900, "output_tokens": 30, "total_tokens": 930, "cache_read_input_tokens": null, "cache_write_input_tokens": null}`),
+			want: protocol.TokenUsageResponse{InputTokens: 900, OutputTokens: 30, APICallCount: 1},
+		},
+		{
+			name: "legitimately zero accumulated_usage stays zero",
+			export: tokenExport(`
+  "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0},
+  "accumulated_usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0}`),
+			want: protocol.TokenUsageResponse{APICallCount: 1},
+		},
+		{
+			name:   "no token fields at all reports zero without error",
+			export: tokenExport(`"accumulated_cost": null`),
+			want:   protocol.TokenUsageResponse{APICallCount: 1},
+		},
+		{
+			name: "cache larger than input never yields negative fresh input",
+			export: tokenExport(`
+  "accumulated_usage": {"input_tokens": 10, "output_tokens": 1, "cache_read_input_tokens": 25}`),
+			want: protocol.TokenUsageResponse{OutputTokens: 1, CacheReadTokens: 25, APICallCount: 1},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := (&Agent{}).CalculateTokens([]byte(tt.export), 0)
+			if err != nil {
+				t.Fatalf("CalculateTokens: %v", err)
+			}
+			if got != tt.want {
+				t.Fatalf("tokens = %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+// Entire calls calculate-tokens on the STORED transcript, not the native
+// export, so the nested totals must survive materialization and stay readable
+// from a scoped slice that has lost the first lines.
+func TestCalculateTokensNestedUsageSurvivesStoredTranscript(t *testing.T) {
+	stored, ok := materializeExport([]byte(nestedUsageExport))
+	if !ok {
+		t.Fatal("nestedUsageExport did not materialize")
+	}
+	want := protocol.TokenUsageResponse{
+		InputTokens: 14, OutputTokens: 394,
+		CacheReadTokens: 112820, CacheCreationTokens: 11777,
+	}
+	for _, tc := range []struct {
+		name     string
+		data     []byte
+		apiCalls int
+	}{
+		{"full stored transcript", stored, 1},
+		{"scoped slice", sliceFromLine(stored, 1), 1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := (&Agent{}).CalculateTokens(tc.data, 0)
+			if err != nil {
+				t.Fatalf("CalculateTokens: %v", err)
+			}
+			want.APICallCount = tc.apiCalls
+			if got != want {
+				t.Fatalf("tokens = %+v, want %+v", got, want)
+			}
+		})
+	}
+}

agents/entire-agent-goose/internal/goose/transcript.go

@@ -15,7 +15,8 @@ import (
 )
 
 // gooseExport models the output of `goose session export --format json`
-// (verified against goose 1.37.0). Message content is camelCase, unlike the
+// (verified against goose 1.37.0; goose 1.39.0+ token fields read per
+// gooseUsage). Message content is camelCase, unlike the
 // snake_case hook payloads. The session-level fields live in gooseSessionMeta,
 // embedded so they stay at the top level of the native document and can also be
 // stamped onto every stored JSONL record (see session_jsonl.go).
@@ -276,10 +277,7 @@ func (a *Agent) CalculateTokens(data []byte, _ int) (protocol.TokenUsageResponse
 	if err != nil {
 		return protocol.TokenUsageResponse{}, err
 	}
-	usage := protocol.TokenUsageResponse{
-		InputTokens:  export.AccumulatedInputTokens,
-		OutputTokens: export.AccumulatedOutput,
-	}
+	usage := sessionTokenUsage(&export.gooseSessionMeta)
 	for _, msg := range export.Conversation {
 		if msg.Role == roleAssistant {
 			usage.APICallCount++
@@ -288,6 +286,54 @@ func (a *Agent) CalculateTokens(data []byte, _ int) (protocol.TokenUsageResponse
 	return usage, nil
 }
 
+// sessionTokenUsage reads whichever token shape the export carries. goose
+// 1.39.0+ writes nested Usage objects: accumulated_usage spans the whole
+// session including compacted history, usage only the current context window,
+// so accumulated_usage wins whenever goose reported it. Older exports carry
+// only the flat accumulated_input/output_tokens fields, with no cache split.
+func sessionTokenUsage(meta *gooseSessionMeta) protocol.TokenUsageResponse {
+	for _, nested := range []*gooseUsage{meta.AccumulatedUsage, meta.Usage} {
+		if nested.reported() {
+			return nested.tokenUsage()
+		}
+	}
+	return protocol.TokenUsageResponse{
+		InputTokens:  meta.AccumulatedInputTokens,
+		OutputTokens: meta.AccumulatedOutput,
+	}
+}
+
+// reported is true when the object carries an input or output count; zeros
+// goose did report count. A missing object, goose's all-null default, or one
+// holding only total/cache figures (which cannot be split into input and
+// output) falls through to the next shape. Fields are never mixed across
+// objects: accumulated_usage and usage cover different spans.
+func (u *gooseUsage) reported() bool {
+	return u != nil && (u.InputTokens != nil || u.OutputTokens != nil)
+}
+
+// tokenUsage maps goose's Usage onto Entire's split. goose's input_tokens
+// counts every prompt token, cache reads and writes included, while Entire's
+// InputTokens is fresh input only and is summed with the cache fields, so the
+// cache share is subtracted to avoid counting it twice.
+func (u *gooseUsage) tokenUsage() protocol.TokenUsageResponse {
+	cacheRead := intValue(u.CacheReadInputTokens)
+	cacheWrite := intValue(u.CacheWriteInputTokens)
+	return protocol.TokenUsageResponse{
+		InputTokens:         max(0, intValue(u.InputTokens)-cacheRead-cacheWrite),
+		CacheCreationTokens: cacheWrite,
+		CacheReadTokens:     cacheRead,
+		OutputTokens:        intValue(u.OutputTokens),
+	}
+}
+
+func intValue(v *int) int {
+	if v == nil {
+		return 0
+	}
+	return *v
+}
+
 func messagesFromOffset(export *gooseExport, offset int) []gooseMessage {
 	if export == nil || offset >= len(export.Conversation) {
 		return nil