chore: track blocked user answer time

This commit is contained in:
2026-04-29 22:55:05 +03:00
parent c3397cd47c
commit 8c0ee461d2
4 changed files with 161 additions and 2 deletions
+137
View File
@@ -97,6 +97,24 @@ const branchConversation = (entries: EntryLike[], maxMessages = 24) => {
return messages.slice(-maxMessages); return messages.slice(-maxMessages);
}; };
const latestAssistantText = (entries: EntryLike[]) => {
for (const entry of [...entries].reverse()) {
if (entry.type !== "message" || entry.message?.role !== "assistant") continue;
const text = extractTextParts(entry.message.content).join("\n").trim();
if (text) return text;
}
return "";
};
const looksBlockedOnUser = (text: string) => {
const clean = text.replace(/\s+/g, " ").trim();
if (!clean) return false;
const directQuestion = /(^|[\s])[^.!?]{8,240}\?\s*($|[\])"'`])/m.test(clean);
const requestForDecision = /\b(please confirm|please provide|which option|what would you prefer|do you want|would you like|should i|can you confirm|could you confirm|waiting for your|i need your)\b/i.test(clean);
return directQuestion || requestForDecision;
};
const slugify = (value: string, fallback = "memory") => { const slugify = (value: string, fallback = "memory") => {
const slug = value const slug = value
.toLowerCase() .toLowerCase()
@@ -154,6 +172,13 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
let activeWorkPausedAt = 0; let activeWorkPausedAt = 0;
let activeWorkAccumulatedMs = 0; let activeWorkAccumulatedMs = 0;
let activeWorkLabel = ""; let activeWorkLabel = "";
let blockedOnUserStartedAt = 0;
let blockedOnUserPrompt = "";
let totalBlockedOnUserMs = 0;
let activeAnswerStartedAt = 0;
let activeAnswerAccumulatedMs = 0;
let activeAnswerLabel = "";
let answerAutoStartedActiveWork = false;
const submittedPrompts: Array<{ at: string; text: string; pauseInclusiveGapMs?: number }> = []; const submittedPrompts: Array<{ at: string; text: string; pauseInclusiveGapMs?: number }> = [];
const baseMetric = (ctx: { cwd: string; sessionManager?: { getSessionFile?: () => string | undefined } }, type: string, data: Record<string, unknown>): SessionMetric => ({ const baseMetric = (ctx: { cwd: string; sessionManager?: { getSessionFile?: () => string | undefined } }, type: string, data: Record<string, unknown>): SessionMetric => ({
@@ -235,6 +260,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
"## Metrics Snapshot", "## Metrics Snapshot",
"", "",
`- Active user work time: ${formatMs(metrics.activeWorkMs as number)}`, `- Active user work time: ${formatMs(metrics.activeWorkMs as number)}`,
`- Active answer time: ${formatMs(metrics.activeAnswerMs as number)}`,
`- Blocked waiting for user: ${formatMs(metrics.blockedOnUserMs as number)}`,
`- Pause-inclusive prompt gaps: ${formatMs(metrics.totalPauseInclusiveGapMs as number)}`, `- Pause-inclusive prompt gaps: ${formatMs(metrics.totalPauseInclusiveGapMs as number)}`,
`- Idle-excluded gap time: ${formatMs(metrics.totalIdleExcludedMs as number)}`, `- Idle-excluded gap time: ${formatMs(metrics.totalIdleExcludedMs as number)}`,
`- Agent duration: ${formatMs(metrics.totalAgentDurationMs as number)}`, `- Agent duration: ${formatMs(metrics.totalAgentDurationMs as number)}`,
@@ -285,11 +312,24 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
return activeWorkAccumulatedMs + Date.now() - activeWorkStartedAt; return activeWorkAccumulatedMs + Date.now() - activeWorkStartedAt;
}; };
const blockedOnUserMs = () => totalBlockedOnUserMs + (blockedOnUserStartedAt ? Date.now() - blockedOnUserStartedAt : 0);
const activeAnswerMs = () => {
if (!activeAnswerStartedAt) return activeAnswerAccumulatedMs;
return activeAnswerAccumulatedMs + Date.now() - activeAnswerStartedAt;
};
const activeWorkSummary = () => ({ const activeWorkSummary = () => ({
activeWorkMs: activeWorkMs(), activeWorkMs: activeWorkMs(),
activeWorkLabel, activeWorkLabel,
activeWorkRunning: activeWorkStartedAt > 0 && activeWorkPausedAt === 0, activeWorkRunning: activeWorkStartedAt > 0 && activeWorkPausedAt === 0,
activeWorkPaused: activeWorkPausedAt > 0, activeWorkPaused: activeWorkPausedAt > 0,
activeAnswerMs: activeAnswerMs(),
activeAnswerLabel,
activeAnswerRunning: activeAnswerStartedAt > 0,
blockedOnUserMs: blockedOnUserMs(),
blockedOnUserActive: blockedOnUserStartedAt > 0,
blockedOnUserPrompt,
promptsSeen: submittedPrompts.length, promptsSeen: submittedPrompts.length,
totalPauseInclusiveGapMs, totalPauseInclusiveGapMs,
totalIdleExcludedMs, totalIdleExcludedMs,
@@ -315,6 +355,10 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
"", "",
`- Active user work time: ${formatMs(summary.activeWorkMs as number)}`, `- Active user work time: ${formatMs(summary.activeWorkMs as number)}`,
`- Active work label: ${summary.activeWorkLabel || "n/a"}`, `- Active work label: ${summary.activeWorkLabel || "n/a"}`,
`- Active answer time: ${formatMs(summary.activeAnswerMs as number)}`,
`- Active answer label: ${summary.activeAnswerLabel || "n/a"}`,
`- Blocked waiting for user: ${formatMs(summary.blockedOnUserMs as number)}`,
`- Currently waiting for user: ${summary.blockedOnUserActive ? "yes" : "no"}`,
`- Prompts submitted: ${summary.promptsSeen}`, `- Prompts submitted: ${summary.promptsSeen}`,
`- Pause-inclusive prompt gaps: ${formatMs(summary.totalPauseInclusiveGapMs as number)}`, `- Pause-inclusive prompt gaps: ${formatMs(summary.totalPauseInclusiveGapMs as number)}`,
`- Idle-excluded gap time: ${formatMs(summary.totalIdleExcludedMs as number)}`, `- Idle-excluded gap time: ${formatMs(summary.totalIdleExcludedMs as number)}`,
@@ -329,6 +373,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
"## Notes", "## Notes",
"", "",
"- Active user work time is measured by explicit `/prompt-start`, `/prompt-pause`, `/prompt-resume`, and `/prompt-stop` commands.", "- Active user work time is measured by explicit `/prompt-start`, `/prompt-pause`, `/prompt-resume`, and `/prompt-stop` commands.",
"- Active answer time is measured by `/answer-start` and `/answer-stop` when you are composing an answer to an agent question.",
"- Blocked waiting for user starts automatically when the last assistant message looks like a direct question and stops on the next interactive user input.",
"- Pi extension APIs do not expose per-keystroke editor activity here, so this is explicit block timing plus automatic idle-capped gap metrics.", "- Pi extension APIs do not expose per-keystroke editor activity here, so this is explicit block timing plus automatic idle-capped gap metrics.",
"- Use LiteLLM and `npx @ccusage/pi@latest session` for provider-side tokens/cost/inference reports.", "- Use LiteLLM and `npx @ccusage/pi@latest session` for provider-side tokens/cost/inference reports.",
"", "",
@@ -360,6 +406,21 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
const now = Date.now(); const now = Date.now();
const pauseInclusiveGapMs = lastAgentEndedAt > 0 ? now - lastAgentEndedAt : undefined; const pauseInclusiveGapMs = lastAgentEndedAt > 0 ? now - lastAgentEndedAt : undefined;
let answeredBlockedPrompt: string | undefined;
let blockedOnUserDurationMs: number | undefined;
if (blockedOnUserStartedAt) {
blockedOnUserDurationMs = now - blockedOnUserStartedAt;
totalBlockedOnUserMs += blockedOnUserDurationMs;
answeredBlockedPrompt = blockedOnUserPrompt;
writeMetric(ctx.cwd, baseMetric(ctx, "blocked_on_user_end", {
blockedOnUserDurationMs,
blockedOnUserPrompt: blockedOnUserPrompt ? truncate(blockedOnUserPrompt, 700) : "",
answerChars: event.text.length,
}));
blockedOnUserStartedAt = 0;
blockedOnUserPrompt = "";
}
promptStartedAt = now; promptStartedAt = now;
currentPrompt = event.text; currentPrompt = event.text;
submittedPrompts.push({ submittedPrompts.push({
@@ -376,6 +437,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
promptChars: event.text.length, promptChars: event.text.length,
pauseInclusiveGapMs, pauseInclusiveGapMs,
idleExcludedMs, idleExcludedMs,
blockedOnUserDurationMs,
answeredBlockedPrompt: answeredBlockedPrompt ? truncate(answeredBlockedPrompt, 500) : undefined,
})); }));
return { action: "continue" }; return { action: "continue" };
@@ -471,6 +534,16 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
captureSnapshot(ctx, "automatic agent_end snapshot"); captureSnapshot(ctx, "automatic agent_end snapshot");
const candidatePath = writeReviewCandidate(ctx, "automatic", "Automatic capture after agent completion. Review before compiling into durable memory."); const candidatePath = writeReviewCandidate(ctx, "automatic", "Automatic capture after agent completion. Review before compiling into durable memory.");
if (ctx.hasUI) ctx.ui.notify(`Memory candidate ready for review: ${relative(ctx.cwd, candidatePath)}`, "info"); if (ctx.hasUI) ctx.ui.notify(`Memory candidate ready for review: ${relative(ctx.cwd, candidatePath)}`, "info");
const finalAssistantText = latestAssistantText(ctx.sessionManager.getBranch());
if (looksBlockedOnUser(finalAssistantText)) {
blockedOnUserStartedAt = now;
blockedOnUserPrompt = truncate(finalAssistantText, 900);
writeMetric(ctx.cwd, baseMetric(ctx, "blocked_on_user_start", {
blockedOnUserPrompt,
}));
if (ctx.hasUI) ctx.ui.notify("Agent appears to be waiting for your answer. Waiting time will stop on your next prompt.", "info");
}
}); });
pi.on("session_before_compact", async (event, ctx) => { pi.on("session_before_compact", async (event, ctx) => {
@@ -502,6 +575,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
`Prompts observed this session: ${submittedPrompts.length}`, `Prompts observed this session: ${submittedPrompts.length}`,
`Provider requests this turn: ${providerRequests}`, `Provider requests this turn: ${providerRequests}`,
`Active user work time: ${formatMs(activeWorkMs())}`, `Active user work time: ${formatMs(activeWorkMs())}`,
`Active answer time: ${formatMs(activeAnswerMs())}`,
`Blocked waiting for user: ${formatMs(blockedOnUserMs())}${blockedOnUserStartedAt ? " (running)" : ""}`,
].join("\n"); ].join("\n");
showOrPrint(ctx, message, "info"); showOrPrint(ctx, message, "info");
@@ -674,6 +749,68 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
}, },
}); });
pi.registerCommand("answer-start", {
description: "Start explicit active answer timer for responding to an agent question",
handler: async (args, ctx) => {
if (activeAnswerStartedAt) {
showOrPrint(ctx, `Active answer timer already running: ${formatMs(activeAnswerMs())}`, "warning");
return;
}
activeAnswerStartedAt = Date.now();
activeAnswerLabel = args.trim() || activeAnswerLabel || "answering agent question";
if (!activeWorkStartedAt && !activeWorkPausedAt) {
activeWorkStartedAt = activeAnswerStartedAt;
activeWorkLabel = activeWorkLabel || activeAnswerLabel;
answerAutoStartedActiveWork = true;
} else {
answerAutoStartedActiveWork = false;
}
writeMetric(ctx.cwd, baseMetric(ctx, "active_answer_start", {
label: activeAnswerLabel,
activeAnswerMs: activeAnswerMs(),
answerAutoStartedActiveWork,
}));
showOrPrint(ctx, `Active answer timer started: ${activeAnswerLabel}`, "success");
},
});
pi.registerCommand("answer-stop", {
description: "Stop explicit active answer timer",
handler: async (_args, ctx) => {
if (!activeAnswerStartedAt) {
showOrPrint(ctx, "Active answer timer is not running.", "warning");
return;
}
activeAnswerAccumulatedMs += Date.now() - activeAnswerStartedAt;
activeAnswerStartedAt = 0;
if (answerAutoStartedActiveWork && activeWorkStartedAt && !activeWorkPausedAt) {
activeWorkAccumulatedMs += Date.now() - activeWorkStartedAt;
activeWorkStartedAt = 0;
activeWorkPausedAt = 0;
}
answerAutoStartedActiveWork = false;
writeMetric(ctx.cwd, baseMetric(ctx, "active_answer_stop", {
...activeWorkSummary(),
}));
showOrPrint(ctx, `Active answer timer stopped at ${formatMs(activeAnswerMs())}`, "success");
},
});
pi.registerCommand("blocked-status", {
description: "Show automatic blocked-on-user timing status",
handler: async (_args, ctx) => {
const message = [
`Blocked waiting for user: ${formatMs(blockedOnUserMs())}`,
`Currently waiting for user: ${blockedOnUserStartedAt ? "yes" : "no"}`,
blockedOnUserPrompt ? `Detected prompt: ${blockedOnUserPrompt}` : "",
].filter(Boolean).join("\n");
showOrPrint(ctx, message, "info");
},
});
pi.registerCommand("time-report", { pi.registerCommand("time-report", {
description: "Write active user work and agent timing report", description: "Write active user work and agent timing report",
handler: async (_args, ctx) => { handler: async (_args, ctx) => {
+12 -2
View File
@@ -57,12 +57,13 @@ Daily entries live in `docs/agent-memory/daily/YYYY-MM-DD.md` when they are safe
The project-local Pi extension lives at `.pi/extensions/agent-memory.ts`. The project-local Pi extension lives at `.pi/extensions/agent-memory.ts`.
It does four things automatically: It does these things automatically:
- injects the reviewed memory index and prompt change log into each agent turn through `before_agent_start` - injects the reviewed memory index and prompt change log into each agent turn through `before_agent_start`
- records private raw metrics and session snapshots under `.agent-memory/raw/` - records private raw metrics and session snapshots under `.agent-memory/raw/`
- creates a reviewable pending memory candidate under `.agent-memory/review/pending/` after each completed agent turn - creates a reviewable pending memory candidate under `.agent-memory/review/pending/` after each completed agent turn
- records provider header latency, agent duration, turn duration, tool counts, prompt submission gaps, and compaction/shutdown events - records provider header latency, agent duration, turn duration, tool counts, prompt submission gaps, and compaction/shutdown events
- detects likely agent questions, starts a blocked-on-user wait timer, and stops it on your next interactive prompt
- exposes memory and timing commands - exposes memory and timing commands
Raw metrics and review candidates are intentionally gitignored. They are evidence for later compilation, not reviewed memory. Raw metrics and review candidates are intentionally gitignored. They are evidence for later compilation, not reviewed memory.
@@ -72,6 +73,8 @@ The extension cannot infer true keystroke-level active typing time from Pi's cur
- `activeWorkMs`: time between `/prompt-start` and `/prompt-stop`, excluding `/prompt-pause` to `/prompt-resume` - `activeWorkMs`: time between `/prompt-start` and `/prompt-stop`, excluding `/prompt-pause` to `/prompt-resume`
- `pauseInclusiveGapMs`: time between previous agent completion and next prompt submission - `pauseInclusiveGapMs`: time between previous agent completion and next prompt submission
- `idleExcludedMs`: the portion above the 5-minute idle cap - `idleExcludedMs`: the portion above the 5-minute idle cap
- `blockedOnUserMs`: waiting time between a detected agent question and your next prompt; this is not counted as active user effort
- `activeAnswerMs`: explicit time between `/answer-start` and `/answer-stop` while you are composing an answer to an agent question
- `agentDurationMs`: prompt submission to `agent_end` - `agentDurationMs`: prompt submission to `agent_end`
- `turnDurationMs`: one LLM/tool turn duration - `turnDurationMs`: one LLM/tool turn duration
- `headerLatencyMs`: provider request to HTTP response headers - `headerLatencyMs`: provider request to HTTP response headers
@@ -95,6 +98,9 @@ Inside Pi:
/prompt-pause /prompt-pause
/prompt-resume /prompt-resume
/prompt-stop /prompt-stop
/answer-start [label]
/answer-stop
/blocked-status
/time-report /time-report
/pi-remember <lesson-or-error-fix> /pi-remember <lesson-or-error-fix>
/pi-memory <question> /pi-memory <question>
@@ -119,7 +125,11 @@ Approve the latest or matching candidate:
Approval moves the candidate to `.agent-memory/review/approved/` and launches `/pi-evolve` so reviewed memory and prompt changes can be proposed. Discard candidates with `/memory-discard` or clear all pending candidates with `/memory-clear`. Approval moves the candidate to `.agent-memory/review/approved/` and launches `/pi-evolve` so reviewed memory and prompt changes can be proposed. Discard candidates with `/memory-discard` or clear all pending candidates with `/memory-clear`.
Use `/prompt-start` when you begin an active prompting/planning block and `/prompt-stop` when you finish. Use `/prompt-pause` before leaving for an extended wait and `/prompt-resume` when you return. `/time-report` writes a private report under `.agent-memory/reports/`. Use `/prompt-start` when you begin an active prompting/planning block and `/prompt-stop` when you finish. Use `/prompt-pause` before leaving for an extended wait and `/prompt-resume` when you return.
When the agent asks a question and you need time to compose the answer, use `/answer-start` before you start thinking/writing and `/answer-stop` after you send the answer. The extension also starts a blocked-on-user wait timer automatically when the final assistant message looks like a direct question. That wait time stops on your next interactive prompt and is reported separately from active user work time.
`/blocked-status` shows whether the automatic blocked-on-user timer is currently running. `/time-report` writes a private report under `.agent-memory/reports/`.
## Guardrails ## Guardrails
+6
View File
@@ -22,3 +22,9 @@
- Added automatic pending memory candidates under `.agent-memory/review/pending/` after each completed agent turn. - Added automatic pending memory candidates under `.agent-memory/review/pending/` after each completed agent turn.
- Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`. - Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`.
- Approval moves a candidate to `.agent-memory/review/approved/` and launches `/pi-evolve`. - Approval moves a candidate to `.agent-memory/review/approved/` and launches `/pi-evolve`.
## [2026-04-29] timing | Blocked user answers
- Added automatic blocked-on-user timing when the final assistant message looks like a direct question.
- Added `/answer-start`, `/answer-stop`, and `/blocked-status`.
- Time reports now separate active user work, active answer composition, and passive waiting for the user.
+6
View File
@@ -25,3 +25,9 @@
- Added reviewable pending memory candidates after each agent turn. - Added reviewable pending memory candidates after each agent turn.
- Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`. - Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`.
- Durable memory and prompt changes still require approval; automatic capture does not write reviewed memory directly. - Durable memory and prompt changes still require approval; automatic capture does not write reviewed memory directly.
## [2026-04-29] timing | Agent questions and answer effort
- Added automatic blocked-on-user detection for direct agent questions.
- Added `/answer-start`, `/answer-stop`, and `/blocked-status`.
- Reports now separate passive wait time from explicit active answer composition.