Mastery tracking v2

python_engine/tokensmith_engine.py

@@ -41,6 +41,8 @@
         keyword_search,
         keyword_terms_for_query,
         list_materials,
+        list_topic_mastery,
+        record_practice_attempt,
         study_documents,
         validate_study_documents,
         practice_source_rows,
@@ -67,6 +69,8 @@
         keyword_search,
         keyword_terms_for_query,
         list_materials,
+        list_topic_mastery,
+        record_practice_attempt,
         study_documents,
         validate_study_documents,
         practice_source_rows,
@@ -2039,6 +2043,23 @@ def resolve_source_document(payload: Dict[str, Any]) -> Dict[str, Any]:
     return {"source": source_document_for_source(payload["userDataPath"], source)}
 
 
+
+def practice_record_attempt(payload: Dict[str, Any]) -> Dict[str, Any]:
+    entry = record_practice_attempt(
+        payload["userDataPath"],
+        session_id=str(payload.get("sessionId") or ""),
+        question_id=str(payload.get("questionId") or ""),
+        topic=str(payload.get("topic") or ""),
+        correct=bool(payload.get("correct")),
+        assisted=bool(payload.get("assisted")),
+    )
+    return {"entry": entry}
+
+
+def practice_list_mastery(payload: Dict[str, Any]) -> Dict[str, Any]:
+    return {"entries": list_topic_mastery(payload["userDataPath"])}
+
+
 COMMANDS = {
     "health": health,
     "preview_cleaning": preview_cleaning,
@@ -2048,6 +2069,8 @@ def resolve_source_document(payload: Dict[str, Any]) -> Dict[str, Any]:
     "starter_sources": starter_sources,
     "study_documents": list_study_documents,
     "practice_sources": practice_sources,
+    "practice_record_attempt": practice_record_attempt,
+    "practice_list_mastery": practice_list_mastery,
     "list_materials": list_indexed_materials,
     "set_material_enabled": set_material_enabled,
     "remove_material": remove_material,

python_engine/tokensmith_store.py

@@ -22,8 +22,12 @@
 
 DB_NAME = "tokensmith.sqlite"
 FAISS_NAME = "tokensmith.faiss"
-SCHEMA_VERSION = 11
+SCHEMA_VERSION = 13
 ALIAS_EXTRACTION_VERSION = 1
+PRACTICE_ASSISTED_CREDIT = 0.5
+# Each newer answer shrinks the weight of older ones by this factor, so mastery
+# follows roughly the last ten answers instead of the whole history.
+PRACTICE_RECENCY_DECAY = 0.9
 KEYWORD_STOPWORDS: Set[str] = {
     "a",
     "about",
@@ -183,6 +187,7 @@ def connect(user_data_path: str) -> sqlite3.Connection:
 def init_db(user_data_path: str) -> None:
     with connect(user_data_path) as conn:
         create_schema(conn)
+        create_practice_schema(conn)
         ensure_column(conn, "tokensmith_collection_state", "embedding_model_id", "TEXT")
         ensure_column(conn, "tokensmith_collection_state", "embedding_model_name", "TEXT")
         ensure_column(conn, "tokensmith_collection_state", "cleaning_profile_id", "TEXT")
@@ -2511,3 +2516,97 @@ def dump_index(user_data_path: str) -> Dict[str, Any]:
         chunks.append(chunk)
 
     return {"version": SCHEMA_VERSION, "documents": documents, "chunks": chunks, "updatedAt": now_iso()}
+
+
+def create_practice_schema(conn: sqlite3.Connection) -> None:
+    columns = {row["name"] for row in conn.execute("PRAGMA table_info(practice_attempts)")}
+    legacy = "verdict" in columns
+    if legacy:
+        # Schema 12 logged every grading detail and kept a separate mastery table.
+        conn.execute("DROP INDEX IF EXISTS idx_practice_attempts_session_id")
+        conn.execute("DROP INDEX IF EXISTS idx_practice_attempts_topic")
+        conn.execute("ALTER TABLE practice_attempts RENAME TO practice_attempts_v12")
+    conn.executescript(
+        """
+        CREATE TABLE IF NOT EXISTS practice_attempts (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id TEXT NOT NULL,
+            question_id TEXT NOT NULL,
+            topic TEXT NOT NULL,
+            correct INTEGER NOT NULL,
+            assisted INTEGER NOT NULL,
+            created_at TEXT NOT NULL,
+            UNIQUE(session_id, question_id)
+        );
+        CREATE INDEX IF NOT EXISTS idx_practice_attempts_topic ON practice_attempts(topic);
+        """
+    )
+    if legacy:
+        conn.execute(
+            """
+            INSERT OR IGNORE INTO practice_attempts(session_id, question_id, topic, correct, assisted, created_at)
+            SELECT session_id, question_id, TRIM(topic), verdict = 'correct', assisted, created_at
+            FROM practice_attempts_v12
+            WHERE verdict IN ('correct', 'incorrect') AND TRIM(COALESCE(topic, '')) != ''
+            ORDER BY id
+            """
+        )
+        conn.execute("DROP TABLE practice_attempts_v12")
+    conn.execute("DROP TABLE IF EXISTS topic_mastery")
+
+
+def _topic_mastery(conn: sqlite3.Connection, topic: Optional[str] = None) -> List[Dict[str, Any]]:
+    rows = conn.execute(
+        f"""
+        SELECT topic, correct, assisted, created_at
+        FROM practice_attempts
+        {"WHERE topic = ?" if topic is not None else ""}
+        ORDER BY topic, id
+        """,
+        ([topic] if topic is not None else []),
+    ).fetchall()
+    totals: Dict[str, Dict[str, Any]] = {}
+    for row in rows:
+        entry = totals.setdefault(row["topic"], {"met": 0.0, "missed": 0.0, "attempts": 0, "updated_at": ""})
+        # Help earns half credit: half counts for the topic and half against it.
+        credit = (PRACTICE_ASSISTED_CREDIT if row["assisted"] else 1.0) if row["correct"] else 0.0
+        entry["met"] = entry["met"] * PRACTICE_RECENCY_DECAY + credit
+        entry["missed"] = entry["missed"] * PRACTICE_RECENCY_DECAY + 1 - credit
+        entry["attempts"] += 1
+        entry["updated_at"] = max(entry["updated_at"], row["created_at"])
+    # One imagined correct and one incorrect answer keep early mastery near 50%.
+    return [
+        {
+            "topic": name,
+            "mastery": (1 + entry["met"]) / (2 + entry["met"] + entry["missed"]),
+            "attempts": entry["attempts"],
+            "updatedAt": entry["updated_at"],
+        }
+        for name, entry in sorted(totals.items())
+    ]
+
+
+def record_practice_attempt(
+    user_data_path: str, session_id: str, question_id: str, topic: str, correct: bool, assisted: bool
+) -> Optional[Dict[str, Any]]:
+    topic = (topic or "").strip()
+    if not topic:
+        return None
+    init_db(user_data_path)
+
+    with connect(user_data_path) as conn:
+        conn.execute(
+            """
+            INSERT OR IGNORE INTO practice_attempts(session_id, question_id, topic, correct, assisted, created_at)
+            VALUES (?, ?, ?, ?, ?, ?)
+            """,
+            (session_id, question_id, topic, 1 if correct else 0, 1 if assisted else 0, now_iso()),
+        )
+        return _topic_mastery(conn, topic)[0]
+
+
+def list_topic_mastery(user_data_path: str) -> List[Dict[str, Any]]:
+    init_db(user_data_path)
+
+    with connect(user_data_path) as conn:
+        return _topic_mastery(conn)

src/main/engine/ollama-service.ts

@@ -113,6 +113,8 @@ interface ActiveOllamaPull {
 }
 
 const activeOllamaPulls = new Map<string, ActiveOllamaPull>()
+const ollamaKeepAlive = '10m'
+const usedOllamaModels = new Map<string, { baseUrl: string; modelName: string }>()
 
 function normalizeOllamaBaseUrl(baseUrl = defaultOllamaBaseUrl): string {
   let normalized = baseUrl.trim().replace(/\/+$/, '')
@@ -863,6 +865,7 @@ async function requestOllamaChatCompletion(
   settings?: ModelRuntimeSettings,
   overrides: OllamaCompletionOverrides = {}
 ): Promise<OllamaChatResponse> {
+  usedOllamaModels.set(`${baseUrl}\n${modelName}`, { baseUrl, modelName })
   if (settings && usesAutomaticContext(settings)) {
     const promptEstimate = Math.ceil(messages.reduce((sum, message) => sum + message.content.length, 0) / 2.5) + 64
     settings = { ...settings, contextLength: localContextForTokens(promptEstimate + (overrides.maxTokens ?? settings.maxLength) + 512,
@@ -885,6 +888,7 @@ async function requestOllamaChatCompletion(
         ...(overrides.temperature !== undefined ? { temperature: overrides.temperature } : {})
       },
       ...(overrides.format ? { format: overrides.format } : {}),
+      keep_alive: ollamaKeepAlive,
       stream: false,
       think: overrides.thinking ?? false
     }),
@@ -898,6 +902,51 @@ async function requestOllamaChatCompletion(
   return (await response.json()) as OllamaChatResponse
 }
 
+export async function warmOllamaModel(model: LocalModel, modelSettings?: ModelRuntimeSettings): Promise<void> {
+  assertOllamaModel(model)
+  const baseUrl = model.ollamaBaseUrl || defaultOllamaBaseUrl
+  usedOllamaModels.set(`${baseUrl}\n${model.ollamaModelName}`, { baseUrl, modelName: model.ollamaModelName })
+  // Refreshing a loaded model with a different num_ctx would make Ollama reload it.
+  const loaded = await loadedOllamaContextLength(baseUrl, model.ollamaModelName)
+  const runtime = loaded ? undefined : (await requestWithOllamaRuntimeContext({ model, modelSettings }, baseUrl, model.ollamaModelName)).modelSettings
+  const contextLength = loaded ?? (runtime && (usesAutomaticContext(runtime) ? localContextForTokens(0, runtime.contextLength) : runtime.contextLength))
+  const response = await fetchWithTimeout(`${ollamaApiBaseUrl(baseUrl)}/chat`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+    body: JSON.stringify({
+      model: model.ollamaModelName,
+      messages: [],
+      keep_alive: ollamaKeepAlive,
+      ...(contextLength ? { options: { num_ctx: contextLength } } : {})
+    })
+  }, 180_000)
+  if (!response.ok) {
+    throw new Error(`Ollama could not load the model: HTTP ${response.status}${await responseErrorDetail(response)}.`)
+  }
+}
+
+async function loadedOllamaContextLength(baseUrl: string, modelName: string): Promise<number | undefined> {
+  try {
+    const response = await fetchWithTimeout(`${ollamaApiBaseUrl(baseUrl)}/ps`, {}, 3_000)
+    if (!response.ok) return undefined
+    const { models = [] } = await response.json() as { models?: (OllamaModelInfo & { context_length?: number })[] }
+    const contextLength = models.find(item => ollamaModelMatches(item, modelName))?.context_length
+    return Number.isInteger(contextLength) && Number(contextLength) > 0 ? Number(contextLength) : undefined
+  } catch {
+    return undefined
+  }
+}
+
+export async function unloadOllamaModels(): Promise<void> {
+  const models = [...usedOllamaModels.values()]
+  usedOllamaModels.clear()
+  await Promise.allSettled(models.map(({ baseUrl, modelName }) => fetchWithTimeout(`${ollamaApiBaseUrl(baseUrl)}/generate`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model: modelName, keep_alive: 0 })
+  }, 3_000)))
+}
+
 async function runOllamaChatCompletion(
   baseUrl: string,
   modelName: string,

src/main/index.ts

@@ -1,5 +1,6 @@
 import type { IndexMaterialOptions } from '../shared/preparation'
-import { preparationReportWithPython } from './python/python-engine-service'
+import type { PracticeAttemptInput } from '../shared/practice'
+import { preparationReportWithPython, writeTokenSmithLog } from './python/python-engine-service'
 import { app, BrowserWindow, dialog, ipcMain, nativeImage, net, safeStorage, shell } from 'electron'
 import { createHash } from 'node:crypto'
 import { existsSync, readFileSync, statSync } from 'node:fs'
@@ -13,6 +14,8 @@ import {
   previewCleaningWithPython,
   studyDocumentsWithPython,
   practiceSourcesWithPython,
+  recordPracticeAttemptWithPython,
+  listTopicMasteryWithPython,
   removeMaterialWithPython,
   readTokenSmithLogFile,
   resolveSourceDocumentWithPython,
@@ -31,7 +34,9 @@ import {
   openOllamaApp,
   openOllamaDownloadPage,
   pullOllamaModel,
-  startOllamaService
+  startOllamaService,
+  unloadOllamaModels,
+  warmOllamaModel
 } from './engine/ollama-service'
 import { searchOllamaLibrary } from './engine/ollama-library-search'
 import { detectDeviceCapabilities } from './system/detect-device-capabilities'
@@ -41,7 +46,7 @@ import {
   modelWithRememberedRemoteApiKey,
   sanitizeAppStateSecrets
 } from './engine/remote-model-secrets'
-import type { AppStateSnapshot, ChatSource, CourseMaterial, LocalModel, LocalModelRole, SearchMode } from '../shared/app-state'
+import type { AppStateSnapshot, ChatSource, CourseMaterial, LocalModel, LocalModelRole, ModelRuntimeSettings, SearchMode } from '../shared/app-state'
 import type { StudyDocumentRef, LibrarySearchOptions } from '../shared/study-scope'
 import { removeRetiredModelState } from '../shared/retired-model-state'
 import type { CleaningProfileId, CleaningRuleId } from '../shared/cleaning'
@@ -530,6 +535,14 @@ app.whenReady().then(async () => {
   ipcMain.handle('library:practice-sources', (_event, documents: StudyDocumentRef[], usedSourceKeys: string[], questionIndex: number) =>
     practiceSourcesWithPython(documents, usedSourceKeys, questionIndex)
   )
+  ipcMain.handle('practice:record-attempt', (_event, input: PracticeAttemptInput) => recordPracticeAttemptWithPython(input))
+  ipcMain.handle('practice:list-mastery', () => listTopicMasteryWithPython())
+  ipcMain.handle('practice:log-question-rejected', (_event, detail: { attempt: number; reason: string; response: string }) =>
+    writeTokenSmithLog('practice_question_validation_failed', detail)
+  )
+  ipcMain.handle('engine:warm-model', (_event, model: LocalModel, modelSettings: ModelRuntimeSettings) =>
+    model.engine === 'ollama' ? warmOllamaModel(model, modelSettings) : undefined
+  )
   ipcMain.handle('library:search', (_event, query: string, materials: CourseMaterial[], limit: number, embeddingModels?: LocalModel[], searchMode?: SearchMode, options?: LibrarySearchOptions) =>
     searchLibraryWithPython(query, materials, limit, embeddingModels, searchMode, options)
   )
@@ -598,6 +611,16 @@ app.whenReady().then(async () => {
   })
 })
 
+let ollamaModelsReleased = false
+app.on('will-quit', event => {
+  if (ollamaModelsReleased) return
+  event.preventDefault()
+  void unloadOllamaModels().finally(() => {
+    ollamaModelsReleased = true
+    app.quit()
+  })
+})
+
 app.on('window-all-closed', () => {
   if (process.platform !== 'darwin') {
     app.quit()

src/main/python/python-engine-service.ts

@@ -12,6 +12,7 @@ import type { ChatSource, CourseMaterial, LocalModel, MaterialIndexProgress, Sea
 import type { CleaningPreviewResult, LibrarySearchResult, TokenSmithLogFile } from '../../shared/engine'
 import type { CleaningProfileId, CleaningRuleId } from '../../shared/cleaning'
 import { normalizeLatencyStageName, sanitizeLatencyDetails, type LatencySpan, type LatencySpanStatus } from '../../shared/latency-trace'
+import type { PracticeAttemptInput, TopicMastery } from '../../shared/practice'
 
 interface PythonRequest {
   id: string
@@ -22,6 +23,8 @@ interface PythonRequest {
     | 'starter_sources'
     | 'study_documents'
     | 'practice_sources'
+    | 'practice_record_attempt'
+    | 'practice_list_mastery'
     | 'list_materials'
     | 'set_material_enabled'
     | 'remove_material'
@@ -835,6 +838,20 @@ export async function practiceSourcesWithPython(documents: StudyDocumentRef[], u
   return result.sources
 }
 
+export async function recordPracticeAttemptWithPython(input: PracticeAttemptInput): Promise<TopicMastery | null> {
+  const result = await requestPython<{ entry: TopicMastery | null }>('practice_record_attempt', {
+    userDataPath: app.getPath('userData'), ...input
+  }, 30_000)
+  return result.entry
+}
+
+export async function listTopicMasteryWithPython(): Promise<TopicMastery[]> {
+  const result = await requestPython<{ entries: TopicMastery[] }>('practice_list_mastery', {
+    userDataPath: app.getPath('userData')
+  }, 30_000)
+  return result.entries
+}
+
 export async function setMaterialEnabledWithPython(materialId: string, isActive: boolean): Promise<void> {
   await requestPython<{ ok: boolean }>(
     'set_material_enabled',

src/preload/index.ts

@@ -47,6 +47,10 @@ const tokenSmithBridge: TokenSmithBridge = {
   studyDocuments: () => ipcRenderer.invoke('library:study-documents'),
   practiceSources: (documents, usedSourceKeys, questionIndex) =>
     ipcRenderer.invoke('library:practice-sources', documents, usedSourceKeys, questionIndex),
+  recordPracticeAttempt: (input) => ipcRenderer.invoke('practice:record-attempt', input),
+  listTopicMastery: () => ipcRenderer.invoke('practice:list-mastery'),
+  logPracticeQuestionRejected: (detail) => ipcRenderer.invoke('practice:log-question-rejected', detail),
+  warmModel: (model, modelSettings) => ipcRenderer.invoke('engine:warm-model', model, modelSettings),
   getPdfForSource: (source) =>
     ipcRenderer.invoke('library:get-pdf-for-source', source) as Promise<
       Awaited<ReturnType<TokenSmithBridge['getPdfForSource']>>

src/renderer/src/App.tsx

@@ -103,6 +103,7 @@ import {
 } from '@shared/model-defaults'
 import { DeviceRecommendationPanel } from './components/DeviceRecommendationPanel'
 import { useDeviceCapabilities } from './hooks/useDeviceCapabilities'
+import { useModelWarmup } from './hooks/useModelWarmup'
 import { modelAwareRetrievalLimit } from '@shared/retrieval-budget'
 import { contextMetadataIdentity, fallbackContextLength, mergeModelContextMetadata, migratedContextMode, normalizeTokenLimit,
   reportedContextLimit, usesAutomaticContext } from '@shared/model-context'
@@ -2487,6 +2488,7 @@ function ChatScreen({
     setChatError(null)
   }, [activeMaterialKey])
   const selectedModelSettings = selectedModel ? modelSettingsFor(settings, selectedModel.id) : settings.modelDefaults
+  useModelWarmup(isActive, selectedModel, selectedModelSettings)
   const canSuggestStarterQuestions =
     isActive &&
     !activeConversation.practiceReference &&