diff --git a/python_engine/tokensmith_engine.py b/python_engine/tokensmith_engine.py index 0caec297..8cc4604b 100644 --- a/python_engine/tokensmith_engine.py +++ b/python_engine/tokensmith_engine.py @@ -41,6 +41,8 @@ keyword_search, keyword_terms_for_query, list_materials, + list_topic_mastery, + record_practice_attempt, study_documents, validate_study_documents, practice_source_rows, @@ -67,6 +69,8 @@ keyword_search, keyword_terms_for_query, list_materials, + list_topic_mastery, + record_practice_attempt, study_documents, validate_study_documents, practice_source_rows, @@ -2039,6 +2043,23 @@ def resolve_source_document(payload: Dict[str, Any]) -> Dict[str, Any]: return {"source": source_document_for_source(payload["userDataPath"], source)} + +def practice_record_attempt(payload: Dict[str, Any]) -> Dict[str, Any]: + entry = record_practice_attempt( + payload["userDataPath"], + session_id=str(payload.get("sessionId") or ""), + question_id=str(payload.get("questionId") or ""), + topic=str(payload.get("topic") or ""), + correct=bool(payload.get("correct")), + assisted=bool(payload.get("assisted")), + ) + return {"entry": entry} + + +def practice_list_mastery(payload: Dict[str, Any]) -> Dict[str, Any]: + return {"entries": list_topic_mastery(payload["userDataPath"])} + + COMMANDS = { "health": health, "preview_cleaning": preview_cleaning, @@ -2048,6 +2069,8 @@ def resolve_source_document(payload: Dict[str, Any]) -> Dict[str, Any]: "starter_sources": starter_sources, "study_documents": list_study_documents, "practice_sources": practice_sources, + "practice_record_attempt": practice_record_attempt, + "practice_list_mastery": practice_list_mastery, "list_materials": list_indexed_materials, "set_material_enabled": set_material_enabled, "remove_material": remove_material, diff --git a/python_engine/tokensmith_store.py b/python_engine/tokensmith_store.py index ba536d70..ac9ca4d9 100644 --- a/python_engine/tokensmith_store.py +++ b/python_engine/tokensmith_store.py @@ -22,8 +22,12 @@ DB_NAME = "tokensmith.sqlite" FAISS_NAME = "tokensmith.faiss" -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 13 ALIAS_EXTRACTION_VERSION = 1 +PRACTICE_ASSISTED_CREDIT = 0.5 +# Each newer answer shrinks the weight of older ones by this factor, so mastery +# follows roughly the last ten answers instead of the whole history. +PRACTICE_RECENCY_DECAY = 0.9 KEYWORD_STOPWORDS: Set[str] = { "a", "about", @@ -183,6 +187,7 @@ def connect(user_data_path: str) -> sqlite3.Connection: def init_db(user_data_path: str) -> None: with connect(user_data_path) as conn: create_schema(conn) + create_practice_schema(conn) ensure_column(conn, "tokensmith_collection_state", "embedding_model_id", "TEXT") ensure_column(conn, "tokensmith_collection_state", "embedding_model_name", "TEXT") ensure_column(conn, "tokensmith_collection_state", "cleaning_profile_id", "TEXT") @@ -2511,3 +2516,97 @@ def dump_index(user_data_path: str) -> Dict[str, Any]: chunks.append(chunk) return {"version": SCHEMA_VERSION, "documents": documents, "chunks": chunks, "updatedAt": now_iso()} + + +def create_practice_schema(conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(practice_attempts)")} + legacy = "verdict" in columns + if legacy: + # Schema 12 logged every grading detail and kept a separate mastery table. + conn.execute("DROP INDEX IF EXISTS idx_practice_attempts_session_id") + conn.execute("DROP INDEX IF EXISTS idx_practice_attempts_topic") + conn.execute("ALTER TABLE practice_attempts RENAME TO practice_attempts_v12") + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS practice_attempts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + question_id TEXT NOT NULL, + topic TEXT NOT NULL, + correct INTEGER NOT NULL, + assisted INTEGER NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(session_id, question_id) + ); + CREATE INDEX IF NOT EXISTS idx_practice_attempts_topic ON practice_attempts(topic); + """ + ) + if legacy: + conn.execute( + """ + INSERT OR IGNORE INTO practice_attempts(session_id, question_id, topic, correct, assisted, created_at) + SELECT session_id, question_id, TRIM(topic), verdict = 'correct', assisted, created_at + FROM practice_attempts_v12 + WHERE verdict IN ('correct', 'incorrect') AND TRIM(COALESCE(topic, '')) != '' + ORDER BY id + """ + ) + conn.execute("DROP TABLE practice_attempts_v12") + conn.execute("DROP TABLE IF EXISTS topic_mastery") + + +def _topic_mastery(conn: sqlite3.Connection, topic: Optional[str] = None) -> List[Dict[str, Any]]: + rows = conn.execute( + f""" + SELECT topic, correct, assisted, created_at + FROM practice_attempts + {"WHERE topic = ?" if topic is not None else ""} + ORDER BY topic, id + """, + ([topic] if topic is not None else []), + ).fetchall() + totals: Dict[str, Dict[str, Any]] = {} + for row in rows: + entry = totals.setdefault(row["topic"], {"met": 0.0, "missed": 0.0, "attempts": 0, "updated_at": ""}) + # Help earns half credit: half counts for the topic and half against it. + credit = (PRACTICE_ASSISTED_CREDIT if row["assisted"] else 1.0) if row["correct"] else 0.0 + entry["met"] = entry["met"] * PRACTICE_RECENCY_DECAY + credit + entry["missed"] = entry["missed"] * PRACTICE_RECENCY_DECAY + 1 - credit + entry["attempts"] += 1 + entry["updated_at"] = max(entry["updated_at"], row["created_at"]) + # One imagined correct and one incorrect answer keep early mastery near 50%. + return [ + { + "topic": name, + "mastery": (1 + entry["met"]) / (2 + entry["met"] + entry["missed"]), + "attempts": entry["attempts"], + "updatedAt": entry["updated_at"], + } + for name, entry in sorted(totals.items()) + ] + + +def record_practice_attempt( + user_data_path: str, session_id: str, question_id: str, topic: str, correct: bool, assisted: bool +) -> Optional[Dict[str, Any]]: + topic = (topic or "").strip() + if not topic: + return None + init_db(user_data_path) + + with connect(user_data_path) as conn: + conn.execute( + """ + INSERT OR IGNORE INTO practice_attempts(session_id, question_id, topic, correct, assisted, created_at) + VALUES (?, ?, ?, ?, ?, ?) + """, + (session_id, question_id, topic, 1 if correct else 0, 1 if assisted else 0, now_iso()), + ) + return _topic_mastery(conn, topic)[0] + + +def list_topic_mastery(user_data_path: str) -> List[Dict[str, Any]]: + init_db(user_data_path) + + with connect(user_data_path) as conn: + return _topic_mastery(conn) diff --git a/src/main/engine/ollama-service.ts b/src/main/engine/ollama-service.ts index 58fa9095..a0455c01 100644 --- a/src/main/engine/ollama-service.ts +++ b/src/main/engine/ollama-service.ts @@ -113,6 +113,8 @@ interface ActiveOllamaPull { } const activeOllamaPulls = new Map() +const ollamaKeepAlive = '10m' +const usedOllamaModels = new Map() function normalizeOllamaBaseUrl(baseUrl = defaultOllamaBaseUrl): string { let normalized = baseUrl.trim().replace(/\/+$/, '') @@ -863,6 +865,7 @@ async function requestOllamaChatCompletion( settings?: ModelRuntimeSettings, overrides: OllamaCompletionOverrides = {} ): Promise { + usedOllamaModels.set(`${baseUrl}\n${modelName}`, { baseUrl, modelName }) if (settings && usesAutomaticContext(settings)) { const promptEstimate = Math.ceil(messages.reduce((sum, message) => sum + message.content.length, 0) / 2.5) + 64 settings = { ...settings, contextLength: localContextForTokens(promptEstimate + (overrides.maxTokens ?? settings.maxLength) + 512, @@ -885,6 +888,7 @@ async function requestOllamaChatCompletion( ...(overrides.temperature !== undefined ? { temperature: overrides.temperature } : {}) }, ...(overrides.format ? { format: overrides.format } : {}), + keep_alive: ollamaKeepAlive, stream: false, think: overrides.thinking ?? false }), @@ -898,6 +902,51 @@ async function requestOllamaChatCompletion( return (await response.json()) as OllamaChatResponse } +export async function warmOllamaModel(model: LocalModel, modelSettings?: ModelRuntimeSettings): Promise { + assertOllamaModel(model) + const baseUrl = model.ollamaBaseUrl || defaultOllamaBaseUrl + usedOllamaModels.set(`${baseUrl}\n${model.ollamaModelName}`, { baseUrl, modelName: model.ollamaModelName }) + // Refreshing a loaded model with a different num_ctx would make Ollama reload it. + const loaded = await loadedOllamaContextLength(baseUrl, model.ollamaModelName) + const runtime = loaded ? undefined : (await requestWithOllamaRuntimeContext({ model, modelSettings }, baseUrl, model.ollamaModelName)).modelSettings + const contextLength = loaded ?? (runtime && (usesAutomaticContext(runtime) ? localContextForTokens(0, runtime.contextLength) : runtime.contextLength)) + const response = await fetchWithTimeout(`${ollamaApiBaseUrl(baseUrl)}/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ + model: model.ollamaModelName, + messages: [], + keep_alive: ollamaKeepAlive, + ...(contextLength ? { options: { num_ctx: contextLength } } : {}) + }) + }, 180_000) + if (!response.ok) { + throw new Error(`Ollama could not load the model: HTTP ${response.status}${await responseErrorDetail(response)}.`) + } +} + +async function loadedOllamaContextLength(baseUrl: string, modelName: string): Promise { + try { + const response = await fetchWithTimeout(`${ollamaApiBaseUrl(baseUrl)}/ps`, {}, 3_000) + if (!response.ok) return undefined + const { models = [] } = await response.json() as { models?: (OllamaModelInfo & { context_length?: number })[] } + const contextLength = models.find(item => ollamaModelMatches(item, modelName))?.context_length + return Number.isInteger(contextLength) && Number(contextLength) > 0 ? Number(contextLength) : undefined + } catch { + return undefined + } +} + +export async function unloadOllamaModels(): Promise { + const models = [...usedOllamaModels.values()] + usedOllamaModels.clear() + await Promise.allSettled(models.map(({ baseUrl, modelName }) => fetchWithTimeout(`${ollamaApiBaseUrl(baseUrl)}/generate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: modelName, keep_alive: 0 }) + }, 3_000))) +} + async function runOllamaChatCompletion( baseUrl: string, modelName: string, diff --git a/src/main/index.ts b/src/main/index.ts index ff4f48da..6d511bfc 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,5 +1,6 @@ import type { IndexMaterialOptions } from '../shared/preparation' -import { preparationReportWithPython } from './python/python-engine-service' +import type { PracticeAttemptInput } from '../shared/practice' +import { preparationReportWithPython, writeTokenSmithLog } from './python/python-engine-service' import { app, BrowserWindow, dialog, ipcMain, nativeImage, net, safeStorage, shell } from 'electron' import { createHash } from 'node:crypto' import { existsSync, readFileSync, statSync } from 'node:fs' @@ -13,6 +14,8 @@ import { previewCleaningWithPython, studyDocumentsWithPython, practiceSourcesWithPython, + recordPracticeAttemptWithPython, + listTopicMasteryWithPython, removeMaterialWithPython, readTokenSmithLogFile, resolveSourceDocumentWithPython, @@ -31,7 +34,9 @@ import { openOllamaApp, openOllamaDownloadPage, pullOllamaModel, - startOllamaService + startOllamaService, + unloadOllamaModels, + warmOllamaModel } from './engine/ollama-service' import { searchOllamaLibrary } from './engine/ollama-library-search' import { detectDeviceCapabilities } from './system/detect-device-capabilities' @@ -41,7 +46,7 @@ import { modelWithRememberedRemoteApiKey, sanitizeAppStateSecrets } from './engine/remote-model-secrets' -import type { AppStateSnapshot, ChatSource, CourseMaterial, LocalModel, LocalModelRole, SearchMode } from '../shared/app-state' +import type { AppStateSnapshot, ChatSource, CourseMaterial, LocalModel, LocalModelRole, ModelRuntimeSettings, SearchMode } from '../shared/app-state' import type { StudyDocumentRef, LibrarySearchOptions } from '../shared/study-scope' import { removeRetiredModelState } from '../shared/retired-model-state' import type { CleaningProfileId, CleaningRuleId } from '../shared/cleaning' @@ -530,6 +535,14 @@ app.whenReady().then(async () => { ipcMain.handle('library:practice-sources', (_event, documents: StudyDocumentRef[], usedSourceKeys: string[], questionIndex: number) => practiceSourcesWithPython(documents, usedSourceKeys, questionIndex) ) + ipcMain.handle('practice:record-attempt', (_event, input: PracticeAttemptInput) => recordPracticeAttemptWithPython(input)) + ipcMain.handle('practice:list-mastery', () => listTopicMasteryWithPython()) + ipcMain.handle('practice:log-question-rejected', (_event, detail: { attempt: number; reason: string; response: string }) => + writeTokenSmithLog('practice_question_validation_failed', detail) + ) + ipcMain.handle('engine:warm-model', (_event, model: LocalModel, modelSettings: ModelRuntimeSettings) => + model.engine === 'ollama' ? warmOllamaModel(model, modelSettings) : undefined + ) ipcMain.handle('library:search', (_event, query: string, materials: CourseMaterial[], limit: number, embeddingModels?: LocalModel[], searchMode?: SearchMode, options?: LibrarySearchOptions) => searchLibraryWithPython(query, materials, limit, embeddingModels, searchMode, options) ) @@ -598,6 +611,16 @@ app.whenReady().then(async () => { }) }) +let ollamaModelsReleased = false +app.on('will-quit', event => { + if (ollamaModelsReleased) return + event.preventDefault() + void unloadOllamaModels().finally(() => { + ollamaModelsReleased = true + app.quit() + }) +}) + app.on('window-all-closed', () => { if (process.platform !== 'darwin') { app.quit() diff --git a/src/main/python/python-engine-service.ts b/src/main/python/python-engine-service.ts index 60e20ec0..61e813f9 100644 --- a/src/main/python/python-engine-service.ts +++ b/src/main/python/python-engine-service.ts @@ -12,6 +12,7 @@ import type { ChatSource, CourseMaterial, LocalModel, MaterialIndexProgress, Sea import type { CleaningPreviewResult, LibrarySearchResult, TokenSmithLogFile } from '../../shared/engine' import type { CleaningProfileId, CleaningRuleId } from '../../shared/cleaning' import { normalizeLatencyStageName, sanitizeLatencyDetails, type LatencySpan, type LatencySpanStatus } from '../../shared/latency-trace' +import type { PracticeAttemptInput, TopicMastery } from '../../shared/practice' interface PythonRequest { id: string @@ -22,6 +23,8 @@ interface PythonRequest { | 'starter_sources' | 'study_documents' | 'practice_sources' + | 'practice_record_attempt' + | 'practice_list_mastery' | 'list_materials' | 'set_material_enabled' | 'remove_material' @@ -835,6 +838,20 @@ export async function practiceSourcesWithPython(documents: StudyDocumentRef[], u return result.sources } +export async function recordPracticeAttemptWithPython(input: PracticeAttemptInput): Promise { + const result = await requestPython<{ entry: TopicMastery | null }>('practice_record_attempt', { + userDataPath: app.getPath('userData'), ...input + }, 30_000) + return result.entry +} + +export async function listTopicMasteryWithPython(): Promise { + const result = await requestPython<{ entries: TopicMastery[] }>('practice_list_mastery', { + userDataPath: app.getPath('userData') + }, 30_000) + return result.entries +} + export async function setMaterialEnabledWithPython(materialId: string, isActive: boolean): Promise { await requestPython<{ ok: boolean }>( 'set_material_enabled', diff --git a/src/preload/index.ts b/src/preload/index.ts index 537a997a..87595748 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -47,6 +47,10 @@ const tokenSmithBridge: TokenSmithBridge = { studyDocuments: () => ipcRenderer.invoke('library:study-documents'), practiceSources: (documents, usedSourceKeys, questionIndex) => ipcRenderer.invoke('library:practice-sources', documents, usedSourceKeys, questionIndex), + recordPracticeAttempt: (input) => ipcRenderer.invoke('practice:record-attempt', input), + listTopicMastery: () => ipcRenderer.invoke('practice:list-mastery'), + logPracticeQuestionRejected: (detail) => ipcRenderer.invoke('practice:log-question-rejected', detail), + warmModel: (model, modelSettings) => ipcRenderer.invoke('engine:warm-model', model, modelSettings), getPdfForSource: (source) => ipcRenderer.invoke('library:get-pdf-for-source', source) as Promise< Awaited> diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 6e883664..9a0964e7 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -103,6 +103,7 @@ import { } from '@shared/model-defaults' import { DeviceRecommendationPanel } from './components/DeviceRecommendationPanel' import { useDeviceCapabilities } from './hooks/useDeviceCapabilities' +import { useModelWarmup } from './hooks/useModelWarmup' import { modelAwareRetrievalLimit } from '@shared/retrieval-budget' import { contextMetadataIdentity, fallbackContextLength, mergeModelContextMetadata, migratedContextMode, normalizeTokenLimit, reportedContextLimit, usesAutomaticContext } from '@shared/model-context' @@ -2487,6 +2488,7 @@ function ChatScreen({ setChatError(null) }, [activeMaterialKey]) const selectedModelSettings = selectedModel ? modelSettingsFor(settings, selectedModel.id) : settings.modelDefaults + useModelWarmup(isActive, selectedModel, selectedModelSettings) const canSuggestStarterQuestions = isActive && !activeConversation.practiceReference && diff --git a/src/renderer/src/PracticeMasteryList.tsx b/src/renderer/src/PracticeMasteryList.tsx new file mode 100644 index 00000000..218f287a --- /dev/null +++ b/src/renderer/src/PracticeMasteryList.tsx @@ -0,0 +1,18 @@ +import { practiceMasteryLabel, practiceMasteryPercent, type TopicMastery } from '@shared/practice' + +export function PracticeMasteryList({ title, entries, disabled, onChoose }: { + title: string; entries: TopicMastery[]; disabled?: boolean; onChoose?: (topic: string) => void +}) { + if (!entries.length) return null + return
+

{title}

+
    {entries.map(entry =>
  • + {onChoose + ? + : {entry.topic}} + {practiceMasteryLabel(entry)} · {entry.attempts} attempt{entry.attempts === 1 ? '' : 's'} + +
  • )}
+
+} diff --git a/src/renderer/src/PracticeScreen.tsx b/src/renderer/src/PracticeScreen.tsx index 6a844bf6..b3c0a432 100644 --- a/src/renderer/src/PracticeScreen.tsx +++ b/src/renderer/src/PracticeScreen.tsx @@ -1,8 +1,9 @@ import { useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { ArrowLeft, ArrowRight, ChartColumn, Plus, RefreshCw } from 'lucide-react' import type { ChatSelectedPassage, ChatSource, CourseMaterial, LocalModel, ModelRuntimeSettings, TokenSmithSettings } from '@shared/app-state' -import { practiceAssisted, practiceMessages, practiceQuestionText, practiceTimeline, practiceReviewLabel, - type PracticeTopic, type PracticeQuestion, type PracticeSession, type PracticeState, type StudyMode } from '@shared/practice' +import { mergeTopicMastery, practiceAssisted, practiceMasteryLabel, practiceMasteryPercent, practiceMasteryWeakestFirst, practiceMessages, + practiceQuestionText, practiceTimeline, practiceTopic, practiceReviewLabel, + type PracticeAttempt, type PracticeTopic, type PracticeQuestion, type PracticeSession, type PracticeState, type StudyMode, type TopicMastery } from '@shared/practice' import { gradeTrueFalse } from '@shared/quiz' import { ChatModelPicker } from './ChatModelPicker' import { MessageText } from './MessageText' @@ -17,7 +18,9 @@ import { PracticeTopicPicker } from './PracticeTopicPicker' import { PracticeActions, PracticeComposer } from './PracticeComposer' import { ConversationViewport } from './ConversationViewport' import { PracticeFeedbackView } from './PracticeFeedbackView' -import { createPracticeClient } from './practice-client' +import { PracticeMasteryList } from './PracticeMasteryList' +import { createPracticeClient, loadTopicMastery, trackPracticeAttempt } from './practice-client' +import { useModelWarmup } from './hooks/useModelWarmup' import type { GenerationProgressApi } from './hooks/useGenerationProgress' import './practice.css' @@ -53,6 +56,7 @@ export function PracticeScreen(props: Props) { const [error, setError] = useState('') const [pending, setPending] = useState<{ id: string } | null>(null) const [reviewOpen, setReviewOpen] = useState(false) + const [mastery, setMastery] = useState([]) const operation = useRef<{ id: string; controller: AbortController; sessionId: string; scopeKey: string } | null>(null) const session = state.sessions.find(item => item.id === state.activeSessionId) const question = session?.questions[session.currentIndex] @@ -64,6 +68,16 @@ export function PracticeScreen(props: Props) { const navigationQuestions = useMemo(() => session?.questions.map(item => ({ id: item.id, text: practiceQuestionText(item) })) ?? [], [session?.questions]) const model = session ? models.find(item => item.id === session.modelId) : props.selectedModel const available = model?.status === 'ready' + useEffect(() => { + if (!isActive) return + let stale = false + void loadTopicMastery().then(entries => { if (!stale) setMastery(entries) }) + return () => { stale = true } + }, [isActive]) + const noteMastery = (entry: TopicMastery | null) => setMastery(current => mergeTopicMastery(current, entry)) + useModelWarmup(isActive, model, model && (session?.modelSettings ?? props.modelSettingsFor(model.id))) + const sessionTopic = session && practiceTopic(session, session.questions[0] ?? { sources: [] }) + const sessionMastery = mastery.find(entry => entry.topic === sessionTopic) const scope = props.scope const scopeKey = studyScopeKey(scope) const selected = scope.map(studyDocumentKey) @@ -172,10 +186,12 @@ export function PracticeScreen(props: Props) { try { const feedback = gradeTrueFalse(question, answer) const id = crypto.randomUUID() + const attempt: PracticeAttempt = { id, answer, feedback, assisted, durationMs: 0 } const checked: PracticeQuestion = { ...question, draft: answer, reviewing: true, attempts: [...question.attempts, - { id, answer, feedback, assisted, durationMs: 0 }], timeline: [...practiceTimeline(question), { kind: 'attempt', id }] } + attempt], timeline: [...practiceTimeline(question), { kind: 'attempt', id }] } const updated = { ...session, questions: session.questions.map(item => item.id === question.id ? checked : item) } updateSession(session.id, () => updated) + void trackPracticeAttempt(session, checked, attempt).then(noteMastery) const nextIndex = updated.questions.findIndex((item, index) => index > updated.currentIndex && item.format === 'true_false') if (nextIndex >= 0) updateSession(session.id, current => ({ ...current, currentIndex: nextIndex })) else if (available && !session.sourcesExhausted) generate(updated) @@ -219,6 +235,7 @@ export function PracticeScreen(props: Props) { aria-pressed={reviewOpen} disabled={Boolean(pending)} onClick={() => setReviewOpen(open => !open)}>