chore: track blocked user answer time

This commit is contained in:
2026-04-29 22:55:05 +03:00
parent c3397cd47c
commit 8c0ee461d2
4 changed files with 161 additions and 2 deletions
+137
View File
@@ -97,6 +97,24 @@ const branchConversation = (entries: EntryLike[], maxMessages = 24) => {
return messages.slice(-maxMessages);
};
const latestAssistantText = (entries: EntryLike[]) => {
for (const entry of [...entries].reverse()) {
if (entry.type !== "message" || entry.message?.role !== "assistant") continue;
const text = extractTextParts(entry.message.content).join("\n").trim();
if (text) return text;
}
return "";
};
const looksBlockedOnUser = (text: string) => {
const clean = text.replace(/\s+/g, " ").trim();
if (!clean) return false;
const directQuestion = /(^|[\s])[^.!?]{8,240}\?\s*($|[\])"'`])/m.test(clean);
const requestForDecision = /\b(please confirm|please provide|which option|what would you prefer|do you want|would you like|should i|can you confirm|could you confirm|waiting for your|i need your)\b/i.test(clean);
return directQuestion || requestForDecision;
};
const slugify = (value: string, fallback = "memory") => {
const slug = value
.toLowerCase()
@@ -154,6 +172,13 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
let activeWorkPausedAt = 0;
let activeWorkAccumulatedMs = 0;
let activeWorkLabel = "";
let blockedOnUserStartedAt = 0;
let blockedOnUserPrompt = "";
let totalBlockedOnUserMs = 0;
let activeAnswerStartedAt = 0;
let activeAnswerAccumulatedMs = 0;
let activeAnswerLabel = "";
let answerAutoStartedActiveWork = false;
const submittedPrompts: Array<{ at: string; text: string; pauseInclusiveGapMs?: number }> = [];
const baseMetric = (ctx: { cwd: string; sessionManager?: { getSessionFile?: () => string | undefined } }, type: string, data: Record<string, unknown>): SessionMetric => ({
@@ -235,6 +260,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
"## Metrics Snapshot",
"",
`- Active user work time: ${formatMs(metrics.activeWorkMs as number)}`,
`- Active answer time: ${formatMs(metrics.activeAnswerMs as number)}`,
`- Blocked waiting for user: ${formatMs(metrics.blockedOnUserMs as number)}`,
`- Pause-inclusive prompt gaps: ${formatMs(metrics.totalPauseInclusiveGapMs as number)}`,
`- Idle-excluded gap time: ${formatMs(metrics.totalIdleExcludedMs as number)}`,
`- Agent duration: ${formatMs(metrics.totalAgentDurationMs as number)}`,
@@ -285,11 +312,24 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
return activeWorkAccumulatedMs + Date.now() - activeWorkStartedAt;
};
const blockedOnUserMs = () => totalBlockedOnUserMs + (blockedOnUserStartedAt ? Date.now() - blockedOnUserStartedAt : 0);
const activeAnswerMs = () => {
if (!activeAnswerStartedAt) return activeAnswerAccumulatedMs;
return activeAnswerAccumulatedMs + Date.now() - activeAnswerStartedAt;
};
const activeWorkSummary = () => ({
activeWorkMs: activeWorkMs(),
activeWorkLabel,
activeWorkRunning: activeWorkStartedAt > 0 && activeWorkPausedAt === 0,
activeWorkPaused: activeWorkPausedAt > 0,
activeAnswerMs: activeAnswerMs(),
activeAnswerLabel,
activeAnswerRunning: activeAnswerStartedAt > 0,
blockedOnUserMs: blockedOnUserMs(),
blockedOnUserActive: blockedOnUserStartedAt > 0,
blockedOnUserPrompt,
promptsSeen: submittedPrompts.length,
totalPauseInclusiveGapMs,
totalIdleExcludedMs,
@@ -315,6 +355,10 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
"",
`- Active user work time: ${formatMs(summary.activeWorkMs as number)}`,
`- Active work label: ${summary.activeWorkLabel || "n/a"}`,
`- Active answer time: ${formatMs(summary.activeAnswerMs as number)}`,
`- Active answer label: ${summary.activeAnswerLabel || "n/a"}`,
`- Blocked waiting for user: ${formatMs(summary.blockedOnUserMs as number)}`,
`- Currently waiting for user: ${summary.blockedOnUserActive ? "yes" : "no"}`,
`- Prompts submitted: ${summary.promptsSeen}`,
`- Pause-inclusive prompt gaps: ${formatMs(summary.totalPauseInclusiveGapMs as number)}`,
`- Idle-excluded gap time: ${formatMs(summary.totalIdleExcludedMs as number)}`,
@@ -329,6 +373,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
"## Notes",
"",
"- Active user work time is measured by explicit `/prompt-start`, `/prompt-pause`, `/prompt-resume`, and `/prompt-stop` commands.",
"- Active answer time is measured by `/answer-start` and `/answer-stop` when you are composing an answer to an agent question.",
"- Blocked waiting for user starts automatically when the last assistant message looks like a direct question and stops on the next interactive user input.",
"- Pi extension APIs do not expose per-keystroke editor activity here, so this is explicit block timing plus automatic idle-capped gap metrics.",
"- Use LiteLLM and `npx @ccusage/pi@latest session` for provider-side tokens/cost/inference reports.",
"",
@@ -360,6 +406,21 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
const now = Date.now();
const pauseInclusiveGapMs = lastAgentEndedAt > 0 ? now - lastAgentEndedAt : undefined;
let answeredBlockedPrompt: string | undefined;
let blockedOnUserDurationMs: number | undefined;
if (blockedOnUserStartedAt) {
blockedOnUserDurationMs = now - blockedOnUserStartedAt;
totalBlockedOnUserMs += blockedOnUserDurationMs;
answeredBlockedPrompt = blockedOnUserPrompt;
writeMetric(ctx.cwd, baseMetric(ctx, "blocked_on_user_end", {
blockedOnUserDurationMs,
blockedOnUserPrompt: blockedOnUserPrompt ? truncate(blockedOnUserPrompt, 700) : "",
answerChars: event.text.length,
}));
blockedOnUserStartedAt = 0;
blockedOnUserPrompt = "";
}
promptStartedAt = now;
currentPrompt = event.text;
submittedPrompts.push({
@@ -376,6 +437,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
promptChars: event.text.length,
pauseInclusiveGapMs,
idleExcludedMs,
blockedOnUserDurationMs,
answeredBlockedPrompt: answeredBlockedPrompt ? truncate(answeredBlockedPrompt, 500) : undefined,
}));
return { action: "continue" };
@@ -471,6 +534,16 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
captureSnapshot(ctx, "automatic agent_end snapshot");
const candidatePath = writeReviewCandidate(ctx, "automatic", "Automatic capture after agent completion. Review before compiling into durable memory.");
if (ctx.hasUI) ctx.ui.notify(`Memory candidate ready for review: ${relative(ctx.cwd, candidatePath)}`, "info");
const finalAssistantText = latestAssistantText(ctx.sessionManager.getBranch());
if (looksBlockedOnUser(finalAssistantText)) {
blockedOnUserStartedAt = now;
blockedOnUserPrompt = truncate(finalAssistantText, 900);
writeMetric(ctx.cwd, baseMetric(ctx, "blocked_on_user_start", {
blockedOnUserPrompt,
}));
if (ctx.hasUI) ctx.ui.notify("Agent appears to be waiting for your answer. Waiting time will stop on your next prompt.", "info");
}
});
pi.on("session_before_compact", async (event, ctx) => {
@@ -502,6 +575,8 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
`Prompts observed this session: ${submittedPrompts.length}`,
`Provider requests this turn: ${providerRequests}`,
`Active user work time: ${formatMs(activeWorkMs())}`,
`Active answer time: ${formatMs(activeAnswerMs())}`,
`Blocked waiting for user: ${formatMs(blockedOnUserMs())}${blockedOnUserStartedAt ? " (running)" : ""}`,
].join("\n");
showOrPrint(ctx, message, "info");
@@ -674,6 +749,68 @@ export default function agentMemoryExtension(pi: ExtensionAPI) {
},
});
pi.registerCommand("answer-start", {
description: "Start explicit active answer timer for responding to an agent question",
handler: async (args, ctx) => {
if (activeAnswerStartedAt) {
showOrPrint(ctx, `Active answer timer already running: ${formatMs(activeAnswerMs())}`, "warning");
return;
}
activeAnswerStartedAt = Date.now();
activeAnswerLabel = args.trim() || activeAnswerLabel || "answering agent question";
if (!activeWorkStartedAt && !activeWorkPausedAt) {
activeWorkStartedAt = activeAnswerStartedAt;
activeWorkLabel = activeWorkLabel || activeAnswerLabel;
answerAutoStartedActiveWork = true;
} else {
answerAutoStartedActiveWork = false;
}
writeMetric(ctx.cwd, baseMetric(ctx, "active_answer_start", {
label: activeAnswerLabel,
activeAnswerMs: activeAnswerMs(),
answerAutoStartedActiveWork,
}));
showOrPrint(ctx, `Active answer timer started: ${activeAnswerLabel}`, "success");
},
});
pi.registerCommand("answer-stop", {
description: "Stop explicit active answer timer",
handler: async (_args, ctx) => {
if (!activeAnswerStartedAt) {
showOrPrint(ctx, "Active answer timer is not running.", "warning");
return;
}
activeAnswerAccumulatedMs += Date.now() - activeAnswerStartedAt;
activeAnswerStartedAt = 0;
if (answerAutoStartedActiveWork && activeWorkStartedAt && !activeWorkPausedAt) {
activeWorkAccumulatedMs += Date.now() - activeWorkStartedAt;
activeWorkStartedAt = 0;
activeWorkPausedAt = 0;
}
answerAutoStartedActiveWork = false;
writeMetric(ctx.cwd, baseMetric(ctx, "active_answer_stop", {
...activeWorkSummary(),
}));
showOrPrint(ctx, `Active answer timer stopped at ${formatMs(activeAnswerMs())}`, "success");
},
});
pi.registerCommand("blocked-status", {
description: "Show automatic blocked-on-user timing status",
handler: async (_args, ctx) => {
const message = [
`Blocked waiting for user: ${formatMs(blockedOnUserMs())}`,
`Currently waiting for user: ${blockedOnUserStartedAt ? "yes" : "no"}`,
blockedOnUserPrompt ? `Detected prompt: ${blockedOnUserPrompt}` : "",
].filter(Boolean).join("\n");
showOrPrint(ctx, message, "info");
},
});
pi.registerCommand("time-report", {
description: "Write active user work and agent timing report",
handler: async (_args, ctx) => {
+12 -2
View File
@@ -57,12 +57,13 @@ Daily entries live in `docs/agent-memory/daily/YYYY-MM-DD.md` when they are safe
The project-local Pi extension lives at `.pi/extensions/agent-memory.ts`.
It does four things automatically:
It does these things automatically:
- injects the reviewed memory index and prompt change log into each agent turn through `before_agent_start`
- records private raw metrics and session snapshots under `.agent-memory/raw/`
- creates a reviewable pending memory candidate under `.agent-memory/review/pending/` after each completed agent turn
- records provider header latency, agent duration, turn duration, tool counts, prompt submission gaps, and compaction/shutdown events
- detects likely agent questions, starts a blocked-on-user wait timer, and stops it on your next interactive prompt
- exposes memory and timing commands
Raw metrics and review candidates are intentionally gitignored. They are evidence for later compilation, not reviewed memory.
@@ -72,6 +73,8 @@ The extension cannot infer true keystroke-level active typing time from Pi's cur
- `activeWorkMs`: time between `/prompt-start` and `/prompt-stop`, excluding `/prompt-pause` to `/prompt-resume`
- `pauseInclusiveGapMs`: time between previous agent completion and next prompt submission
- `idleExcludedMs`: the portion above the 5-minute idle cap
- `blockedOnUserMs`: waiting time between a detected agent question and your next prompt; this is not counted as active user effort
- `activeAnswerMs`: explicit time between `/answer-start` and `/answer-stop` while you are composing an answer to an agent question
- `agentDurationMs`: prompt submission to `agent_end`
- `turnDurationMs`: one LLM/tool turn duration
- `headerLatencyMs`: provider request to HTTP response headers
@@ -95,6 +98,9 @@ Inside Pi:
/prompt-pause
/prompt-resume
/prompt-stop
/answer-start [label]
/answer-stop
/blocked-status
/time-report
/pi-remember <lesson-or-error-fix>
/pi-memory <question>
@@ -119,7 +125,11 @@ Approve the latest or matching candidate:
Approval moves the candidate to `.agent-memory/review/approved/` and launches `/pi-evolve` so reviewed memory and prompt changes can be proposed. Discard candidates with `/memory-discard` or clear all pending candidates with `/memory-clear`.
Use `/prompt-start` when you begin an active prompting/planning block and `/prompt-stop` when you finish. Use `/prompt-pause` before leaving for an extended wait and `/prompt-resume` when you return. `/time-report` writes a private report under `.agent-memory/reports/`.
Use `/prompt-start` when you begin an active prompting/planning block and `/prompt-stop` when you finish. Use `/prompt-pause` before leaving for an extended wait and `/prompt-resume` when you return.
When the agent asks a question and you need time to compose the answer, use `/answer-start` before you start thinking/writing and `/answer-stop` after you send the answer. The extension also starts a blocked-on-user wait timer automatically when the final assistant message looks like a direct question. That wait time stops on your next interactive prompt and is reported separately from active user work time.
`/blocked-status` shows whether the automatic blocked-on-user timer is currently running. `/time-report` writes a private report under `.agent-memory/reports/`.
## Guardrails
+6
View File
@@ -22,3 +22,9 @@
- Added automatic pending memory candidates under `.agent-memory/review/pending/` after each completed agent turn.
- Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`.
- Approval moves a candidate to `.agent-memory/review/approved/` and launches `/pi-evolve`.
## [2026-04-29] timing | Blocked user answers
- Added automatic blocked-on-user timing when the final assistant message looks like a direct question.
- Added `/answer-start`, `/answer-stop`, and `/blocked-status`.
- Time reports now separate active user work, active answer composition, and passive waiting for the user.
+6
View File
@@ -25,3 +25,9 @@
- Added reviewable pending memory candidates after each agent turn.
- Added `/memory-review`, `/memory-show`, `/memory-approve`, `/memory-discard`, and `/memory-clear`.
- Durable memory and prompt changes still require approval; automatic capture does not write reviewed memory directly.
## [2026-04-29] timing | Agent questions and answer effort
- Added automatic blocked-on-user detection for direct agent questions.
- Added `/answer-start`, `/answer-stop`, and `/blocked-status`.
- Reports now separate passive wait time from explicit active answer composition.