From 8c0ee461d2e0ee8a685031b6706032947498f735 Mon Sep 17 00:00:00 2001 From: gnezim Date: Wed, 29 Apr 2026 22:55:05 +0300 Subject: [PATCH] chore: track blocked user answer time --- .pi/extensions/agent-memory.ts | 137 +++++++++++++++++++++++++ docs/agent-memory/README.md | 14 ++- docs/agent-memory/log.md | 6 ++ docs/agent-memory/prompt-change-log.md | 6 ++ 4 files changed, 161 insertions(+), 2 deletions(-) diff --git a/.pi/extensions/agent-memory.ts b/.pi/extensions/agent-memory.ts index c6898b65..e8825570 100644 --- a/.pi/extensions/agent-memory.ts +++ b/.pi/extensions/agent-memory.ts @@ -97,6 +97,24 @@ const branchConversation = (entries: EntryLike[], maxMessages = 24) => { return messages.slice(-maxMessages); }; +const latestAssistantText = (entries: EntryLike[]) => { + for (const entry of [...entries].reverse()) { + if (entry.type !== "message" || entry.message?.role !== "assistant") continue; + const text = extractTextParts(entry.message.content).join("\n").trim(); + if (text) return text; + } + return ""; +}; + +const looksBlockedOnUser = (text: string) => { + const clean = text.replace(/\s+/g, " ").trim(); + if (!clean) return false; + + const directQuestion = /(^|[\s])[^.!?]{8,240}\?\s*($|[\])"'`])/m.test(clean); + const requestForDecision = /\b(please confirm|please provide|which option|what would you prefer|do you want|would you like|should i|can you confirm|could you confirm|waiting for your|i need your)\b/i.test(clean); + return directQuestion || requestForDecision; +}; + const slugify = (value: string, fallback = "memory") => { const slug = value .toLowerCase() @@ -154,6 +172,13 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { let activeWorkPausedAt = 0; let activeWorkAccumulatedMs = 0; let activeWorkLabel = ""; + let blockedOnUserStartedAt = 0; + let blockedOnUserPrompt = ""; + let totalBlockedOnUserMs = 0; + let activeAnswerStartedAt = 0; + let activeAnswerAccumulatedMs = 0; + let activeAnswerLabel = ""; + let answerAutoStartedActiveWork = false; const submittedPrompts: Array<{ at: string; text: string; pauseInclusiveGapMs?: number }> = []; const baseMetric = (ctx: { cwd: string; sessionManager?: { getSessionFile?: () => string | undefined } }, type: string, data: Record): SessionMetric => ({ @@ -235,6 +260,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { "## Metrics Snapshot", "", `- Active user work time: ${formatMs(metrics.activeWorkMs as number)}`, + `- Active answer time: ${formatMs(metrics.activeAnswerMs as number)}`, + `- Blocked waiting for user: ${formatMs(metrics.blockedOnUserMs as number)}`, `- Pause-inclusive prompt gaps: ${formatMs(metrics.totalPauseInclusiveGapMs as number)}`, `- Idle-excluded gap time: ${formatMs(metrics.totalIdleExcludedMs as number)}`, `- Agent duration: ${formatMs(metrics.totalAgentDurationMs as number)}`, @@ -285,11 +312,24 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { return activeWorkAccumulatedMs + Date.now() - activeWorkStartedAt; }; + const blockedOnUserMs = () => totalBlockedOnUserMs + (blockedOnUserStartedAt ? Date.now() - blockedOnUserStartedAt : 0); + + const activeAnswerMs = () => { + if (!activeAnswerStartedAt) return activeAnswerAccumulatedMs; + return activeAnswerAccumulatedMs + Date.now() - activeAnswerStartedAt; + }; + const activeWorkSummary = () => ({ activeWorkMs: activeWorkMs(), activeWorkLabel, activeWorkRunning: activeWorkStartedAt > 0 && activeWorkPausedAt === 0, activeWorkPaused: activeWorkPausedAt > 0, + activeAnswerMs: activeAnswerMs(), + activeAnswerLabel, + activeAnswerRunning: activeAnswerStartedAt > 0, + blockedOnUserMs: blockedOnUserMs(), + blockedOnUserActive: blockedOnUserStartedAt > 0, + blockedOnUserPrompt, promptsSeen: submittedPrompts.length, totalPauseInclusiveGapMs, totalIdleExcludedMs, @@ -315,6 +355,10 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { "", `- Active user work time: ${formatMs(summary.activeWorkMs as number)}`, `- Active work label: ${summary.activeWorkLabel || "n/a"}`, + `- Active answer time: ${formatMs(summary.activeAnswerMs as number)}`, + `- Active answer label: ${summary.activeAnswerLabel || "n/a"}`, + `- Blocked waiting for user: ${formatMs(summary.blockedOnUserMs as number)}`, + `- Currently waiting for user: ${summary.blockedOnUserActive ? "yes" : "no"}`, `- Prompts submitted: ${summary.promptsSeen}`, `- Pause-inclusive prompt gaps: ${formatMs(summary.totalPauseInclusiveGapMs as number)}`, `- Idle-excluded gap time: ${formatMs(summary.totalIdleExcludedMs as number)}`, @@ -329,6 +373,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { "## Notes", "", "- Active user work time is measured by explicit `/prompt-start`, `/prompt-pause`, `/prompt-resume`, and `/prompt-stop` commands.", + "- Active answer time is measured by `/answer-start` and `/answer-stop` when you are composing an answer to an agent question.", + "- Blocked waiting for user starts automatically when the last assistant message looks like a direct question and stops on the next interactive user input.", "- Pi extension APIs do not expose per-keystroke editor activity here, so this is explicit block timing plus automatic idle-capped gap metrics.", "- Use LiteLLM and `npx @ccusage/pi@latest session` for provider-side tokens/cost/inference reports.", "", @@ -360,6 +406,21 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { const now = Date.now(); const pauseInclusiveGapMs = lastAgentEndedAt > 0 ? now - lastAgentEndedAt : undefined; + let answeredBlockedPrompt: string | undefined; + let blockedOnUserDurationMs: number | undefined; + if (blockedOnUserStartedAt) { + blockedOnUserDurationMs = now - blockedOnUserStartedAt; + totalBlockedOnUserMs += blockedOnUserDurationMs; + answeredBlockedPrompt = blockedOnUserPrompt; + writeMetric(ctx.cwd, baseMetric(ctx, "blocked_on_user_end", { + blockedOnUserDurationMs, + blockedOnUserPrompt: blockedOnUserPrompt ? truncate(blockedOnUserPrompt, 700) : "", + answerChars: event.text.length, + })); + blockedOnUserStartedAt = 0; + blockedOnUserPrompt = ""; + } + promptStartedAt = now; currentPrompt = event.text; submittedPrompts.push({ @@ -376,6 +437,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { promptChars: event.text.length, pauseInclusiveGapMs, idleExcludedMs, + blockedOnUserDurationMs, + answeredBlockedPrompt: answeredBlockedPrompt ? truncate(answeredBlockedPrompt, 500) : undefined, })); return { action: "continue" }; @@ -471,6 +534,16 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { captureSnapshot(ctx, "automatic agent_end snapshot"); const candidatePath = writeReviewCandidate(ctx, "automatic", "Automatic capture after agent completion. Review before compiling into durable memory."); if (ctx.hasUI) ctx.ui.notify(`Memory candidate ready for review: ${relative(ctx.cwd, candidatePath)}`, "info"); + + const finalAssistantText = latestAssistantText(ctx.sessionManager.getBranch()); + if (looksBlockedOnUser(finalAssistantText)) { + blockedOnUserStartedAt = now; + blockedOnUserPrompt = truncate(finalAssistantText, 900); + writeMetric(ctx.cwd, baseMetric(ctx, "blocked_on_user_start", { + blockedOnUserPrompt, + })); + if (ctx.hasUI) ctx.ui.notify("Agent appears to be waiting for your answer. Waiting time will stop on your next prompt.", "info"); + } }); pi.on("session_before_compact", async (event, ctx) => { @@ -502,6 +575,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { `Prompts observed this session: ${submittedPrompts.length}`, `Provider requests this turn: ${providerRequests}`, `Active user work time: ${formatMs(activeWorkMs())}`, + `Active answer time: ${formatMs(activeAnswerMs())}`, + `Blocked waiting for user: ${formatMs(blockedOnUserMs())}${blockedOnUserStartedAt ? " (running)" : ""}`, ].join("\n"); showOrPrint(ctx, message, "info"); @@ -674,6 +749,68 @@ export default function agentMemoryExtension(pi: ExtensionAPI) { }, }); + pi.registerCommand("answer-start", { + description: "Start explicit active answer timer for responding to an agent question", + handler: async (args, ctx) => { + if (activeAnswerStartedAt) { + showOrPrint(ctx, `Active answer timer already running: ${formatMs(activeAnswerMs())}`, "warning"); + return; + } + + activeAnswerStartedAt = Date.now(); + activeAnswerLabel = args.trim() || activeAnswerLabel || "answering agent question"; + if (!activeWorkStartedAt && !activeWorkPausedAt) { + activeWorkStartedAt = activeAnswerStartedAt; + activeWorkLabel = activeWorkLabel || activeAnswerLabel; + answerAutoStartedActiveWork = true; + } else { + answerAutoStartedActiveWork = false; + } + + writeMetric(ctx.cwd, baseMetric(ctx, "active_answer_start", { + label: activeAnswerLabel, + activeAnswerMs: activeAnswerMs(), + answerAutoStartedActiveWork, + })); + showOrPrint(ctx, `Active answer timer started: ${activeAnswerLabel}`, "success"); + }, + }); + + pi.registerCommand("answer-stop", { + description: "Stop explicit active answer timer", + handler: async (_args, ctx) => { + if (!activeAnswerStartedAt) { + showOrPrint(ctx, "Active answer timer is not running.", "warning"); + return; + } + + activeAnswerAccumulatedMs += Date.now() - activeAnswerStartedAt; + activeAnswerStartedAt = 0; + if (answerAutoStartedActiveWork && activeWorkStartedAt && !activeWorkPausedAt) { + activeWorkAccumulatedMs += Date.now() - activeWorkStartedAt; + activeWorkStartedAt = 0; + activeWorkPausedAt = 0; + } + answerAutoStartedActiveWork = false; + writeMetric(ctx.cwd, baseMetric(ctx, "active_answer_stop", { + ...activeWorkSummary(), + })); + showOrPrint(ctx, `Active answer timer stopped at ${formatMs(activeAnswerMs())}`, "success"); + }, + }); + + pi.registerCommand("blocked-status", { + description: "Show automatic blocked-on-user timing status", + handler: async (_args, ctx) => { + const message = [ + `Blocked waiting for user: ${formatMs(blockedOnUserMs())}`, + `Currently waiting for user: ${blockedOnUserStartedAt ? "yes" : "no"}`, + blockedOnUserPrompt ? `Detected prompt: ${blockedOnUserPrompt}` : "", + ].filter(Boolean).join("\n"); + showOrPrint(ctx, message, "info"); + }, + }); + pi.registerCommand("time-report", { description: "Write active user work and agent timing report", handler: async (_args, ctx) => { diff --git a/docs/agent-memory/README.md b/docs/agent-memory/README.md index c2b61b97..66e79309 100644 --- a/docs/agent-memory/README.md +++ b/docs/agent-memory/README.md @@ -57,12 +57,13 @@ Daily entries live in `docs/agent-memory/daily/YYYY-MM-DD.md` when they are safe The project-local Pi extension lives at `.pi/extensions/agent-memory.ts`. -It does four things automatically: +It does these things automatically: - injects the reviewed memory index and prompt change log into each agent turn through `before_agent_start` - records private raw metrics and session snapshots under `.agent-memory/raw/` - creates a reviewable pending memory candidate under `.agent-memory/review/pending/` after each completed agent turn - records provider header latency, agent duration, turn duration, tool counts, prompt submission gaps, and compaction/shutdown events +- detects likely agent questions, starts a blocked-on-user wait timer, and stops it on your next interactive prompt - exposes memory and timing commands Raw metrics and review candidates are intentionally gitignored. They are evidence for later compilation, not reviewed memory. @@ -72,6 +73,8 @@ The extension cannot infer true keystroke-level active typing time from Pi's cur - `activeWorkMs`: time between `/prompt-start` and `/prompt-stop`, excluding `/prompt-pause` to `/prompt-resume` - `pauseInclusiveGapMs`: time between previous agent completion and next prompt submission - `idleExcludedMs`: the portion above the 5-minute idle cap +- `blockedOnUserMs`: waiting time between a detected agent question and your next prompt; this is not counted as active user effort +- `activeAnswerMs`: explicit time between `/answer-start` and `/answer-stop` while you are composing an answer to an agent question - `agentDurationMs`: prompt submission to `agent_end` - `turnDurationMs`: one LLM/tool turn duration - `headerLatencyMs`: provider request to HTTP response headers @@ -95,6 +98,9 @@ Inside Pi: /prompt-pause /prompt-resume /prompt-stop +/answer-start [label] +/answer-stop +/blocked-status /time-report /pi-remember /pi-memory @@ -119,7 +125,11 @@ Approve the latest or matching candidate: Approval moves the candidate to `.agent-memory/review/approved/` and launches `/pi-evolve` so reviewed memory and prompt changes can be proposed. Discard candidates with `/memory-discard` or clear all pending candidates with `/memory-clear`. -Use `/prompt-start` when you begin an active prompting/planning block and `/prompt-stop` when you finish. Use `/prompt-pause` before leaving for an extended wait and `/prompt-resume` when you return. `/time-report` writes a private report under `.agent-memory/reports/`. +Use `/prompt-start` when you begin an active prompting/planning block and `/prompt-stop` when you finish. Use `/prompt-pause` before leaving for an extended wait and `/prompt-resume` when you return. + +When the agent asks a question and you need time to compose the answer, use `/answer-start` before you start thinking/writing and `/answer-stop` after you send the answer. The extension also starts a blocked-on-user wait timer automatically when the final assistant message looks like a direct question. That wait time stops on your next interactive prompt and is reported separately from active user work time. + +`/blocked-status` shows whether the automatic blocked-on-user timer is currently running. `/time-report` writes a private report under `.agent-memory/reports/`. ## Guardrails diff --git a/docs/agent-memory/log.md b/docs/agent-memory/log.md index 0546bcd0..30896bb0 100644 --- a/docs/agent-memory/log.md +++ b/docs/agent-memory/log.md @@ -22,3 +22,9 @@ - Added automatic pending memory candidates under `.agent-memory/review/pending/` after each completed agent turn. - Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`. - Approval moves a candidate to `.agent-memory/review/approved/` and launches `/pi-evolve`. + +## [2026-04-29] timing | Blocked user answers + +- Added automatic blocked-on-user timing when the final assistant message looks like a direct question. +- Added `/answer-start`, `/answer-stop`, and `/blocked-status`. +- Time reports now separate active user work, active answer composition, and passive waiting for the user. diff --git a/docs/agent-memory/prompt-change-log.md b/docs/agent-memory/prompt-change-log.md index fd1eaff7..b5a1ed57 100644 --- a/docs/agent-memory/prompt-change-log.md +++ b/docs/agent-memory/prompt-change-log.md @@ -25,3 +25,9 @@ - Added reviewable pending memory candidates after each agent turn. - Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`. - Durable memory and prompt changes still require approval; automatic capture does not write reviewed memory directly. + +## [2026-04-29] timing | Agent questions and answer effort + +- Added automatic blocked-on-user detection for direct agent questions. +- Added `/answer-start`, `/answer-stop`, and `/blocked-status`. +- Reports now separate passive wait time from explicit active answer composition.