From 38f0560f9b39c2115021e816bc11fc4418967ccb Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 29 May 2026 14:51:14 +0800 Subject: [PATCH] Syntax-highlight Edit, Write, and Read tool calls (#1218) Highlighting reuses the existing code-block tokenizer and LRU cache, runs lazily on expand, and falls back to plain text above a size cap so a large Read can't stall the main thread. The diff highlighter reconstructs the old and new file text by position rather than from @@ line ranges, so diffs that arrive without real hunk headers still highlight. Read content is normalized server-side: the cat -n line-number gutter some providers emit is stripped so content is uniformly raw source, with the first line number surfaced as offset for the client to rebuild the gutter. --- packages/app/src/components/diff-viewer.tsx | 38 +++++++ .../src/components/highlighted-code-block.tsx | 71 +----------- .../src/components/highlighted-content.tsx | 104 ++++++++++++++++++ .../app/src/components/tool-call-details.tsx | 53 +++++++-- packages/app/src/utils/diff-highlight.test.ts | 53 +++++++++ packages/app/src/utils/diff-highlight.ts | 91 +++++++++++++++ .../app/src/utils/highlight-cache.test.ts | 65 +++++++++++ packages/app/src/utils/highlight-cache.ts | 89 +++++++++++++++ packages/app/src/utils/tool-call-parsers.ts | 4 + .../providers/tool-call-detail-primitives.ts | 9 +- .../agent/providers/tool-call-mapper-utils.ts | 75 +++++++++++++ .../providers/tool-call-read-gutter.test.ts | 86 +++++++++++++++ 12 files changed, 661 insertions(+), 77 deletions(-) create mode 100644 packages/app/src/components/highlighted-content.tsx create mode 100644 packages/app/src/utils/diff-highlight.test.ts create mode 100644 packages/app/src/utils/diff-highlight.ts create mode 100644 packages/app/src/utils/highlight-cache.test.ts create mode 100644 packages/app/src/utils/highlight-cache.ts create mode 100644 packages/server/src/server/agent/providers/tool-call-read-gutter.test.ts diff --git a/packages/app/src/components/diff-viewer.tsx b/packages/app/src/components/diff-viewer.tsx index a44e7808d..825c13bf5 100644 --- a/packages/app/src/components/diff-viewer.tsx +++ b/packages/app/src/components/diff-viewer.tsx @@ -4,6 +4,8 @@ import { ScrollView as GHScrollView } from "react-native-gesture-handler"; import { StyleSheet } from "react-native-unistyles"; import { Fonts } from "@/constants/theme"; import type { DiffLine } from "@/utils/tool-call-parsers"; +import { diffLinePrefix } from "@/utils/diff-highlight"; +import { syntaxTokenStyleFor } from "@/styles/syntax-token-styles"; import { useWebScrollbarStyle } from "@/hooks/use-web-scrollbar-style"; import { inlineUnistylesStyle } from "@/styles/unistyles-inline-style"; import { getCodeInsets } from "./code-insets"; @@ -40,6 +42,26 @@ function DiffLineRow({ line }: { line: DiffLine }) { [line.type], ); + const prefixStyle = React.useMemo( + () => [ + line.type === "add" && styles.addText, + line.type === "remove" && styles.removeText, + line.type === "context" && styles.contextText, + ], + [line.type], + ); + + if (line.tokens) { + return ( + + + {diffLinePrefix(line)} + + + + ); + } + return ( {line.segments ? ( @@ -62,6 +84,22 @@ function DiffLineRow({ line }: { line: DiffLine }) { ); } +function DiffTokens({ tokens }: { tokens: NonNullable }) { + const keyed = React.useMemo( + () => tokens.map((token, index) => ({ key: `${index}-${token.text}`, token })), + [tokens], + ); + return ( + <> + {keyed.map(({ key, token }) => ( + + {token.text} + + ))} + + ); +} + function DiffSegment({ segment, lineType, diff --git a/packages/app/src/components/highlighted-code-block.tsx b/packages/app/src/components/highlighted-code-block.tsx index 04a04bd2e..792f1740d 100644 --- a/packages/app/src/components/highlighted-code-block.tsx +++ b/packages/app/src/components/highlighted-code-block.tsx @@ -4,10 +4,11 @@ import { StyleSheet } from "react-native-unistyles"; import { MarkdownTextSpan } from "@/components/markdown-text"; import * as Clipboard from "expo-clipboard"; import { Check, Copy } from "lucide-react-native"; -import { highlightCode, type HighlightToken } from "@getpaseo/highlight"; +import type { HighlightToken } from "@getpaseo/highlight"; import { isNative, isWeb } from "@/constants/platform"; import { useIsCompactFormFactor } from "@/constants/layout"; import { syntaxTokenStyleFor } from "@/styles/syntax-token-styles"; +import { highlightToKeyedLines, type KeyedLine } from "@/utils/highlight-cache"; interface HighlightedCodeBlockProps { code: string; @@ -44,34 +45,6 @@ function stripTerminalFenceNewline(code: string): string { return code.endsWith("\n") ? code.slice(0, -1) : code; } -// Cross-instance cache for tokenized code blocks. Tokenization is -// theme-independent (colors are applied at render time), so the key is just -// (language, code). Bounded by entry count — 200 is generous for a chat -// transcript, code blocks rarely repeat beyond a handful of distinct shapes. -class LRUCache { - private readonly map = new Map(); - constructor(private readonly max: number) {} - - get(key: K): V | undefined { - const value = this.map.get(key); - if (value === undefined) return undefined; - this.map.delete(key); - this.map.set(key, value); - return value; - } - - set(key: K, value: V): void { - if (this.map.has(key)) this.map.delete(key); - else if (this.map.size >= this.max) { - const oldest = this.map.keys().next().value; - if (oldest !== undefined) this.map.delete(oldest); - } - this.map.set(key, value); - } -} - -const tokenizationCache = new LRUCache(200); - export const HighlightedCodeBlock = React.memo(function HighlightedCodeBlock({ code, language, @@ -87,22 +60,10 @@ export const HighlightedCodeBlock = React.memo(function HighlightedCodeBlock({ ); const renderedCode = useMemo(() => stripTerminalFenceNewline(code), [code]); - const keyedLines = useMemo(() => { - const ext = fenceLanguageToExtension(language); - if (!ext) return null; - const cacheKey = `${ext}:${renderedCode}`; - const cached = tokenizationCache.get(cacheKey); - if (cached) return cached; - let tokenizedLines: HighlightToken[][]; - try { - tokenizedLines = highlightCode(renderedCode, `x.${ext}`); - } catch { - return null; - } - const result = tokenizedLines.map(toKeyedLine); - tokenizationCache.set(cacheKey, result); - return result; - }, [renderedCode, language]); + const keyedLines = useMemo( + () => highlightToKeyedLines(renderedCode, fenceLanguageToExtension(language)), + [renderedCode, language], + ); const isCompact = useIsCompactFormFactor(); const [isHovered, setIsHovered] = useState(false); @@ -127,26 +88,6 @@ export const HighlightedCodeBlock = React.memo(function HighlightedCodeBlock({ ); }); -interface KeyedToken { - key: string; - token: HighlightToken; -} - -interface KeyedLine { - key: string; - tokens: KeyedToken[]; -} - -function toKeyedLine(tokens: HighlightToken[], lineIndex: number): KeyedLine { - return { - key: `line-${lineIndex}`, - tokens: tokens.map((token, tokenIndex) => ({ - key: `${lineIndex}-${tokenIndex}`, - token, - })), - }; -} - function renderCodeSegments(keyedLines: KeyedLine[]): React.ReactNode[] { const segments: React.ReactNode[] = []; for (let lineIndex = 0; lineIndex < keyedLines.length; lineIndex += 1) { diff --git a/packages/app/src/components/highlighted-content.tsx b/packages/app/src/components/highlighted-content.tsx new file mode 100644 index 000000000..0dc85c369 --- /dev/null +++ b/packages/app/src/components/highlighted-content.tsx @@ -0,0 +1,104 @@ +import React from "react"; +import { Text, View } from "react-native"; +import { StyleSheet } from "react-native-unistyles"; +import { Fonts } from "@/constants/theme"; +import { isWeb } from "@/constants/platform"; +import { syntaxTokenStyleFor } from "@/styles/syntax-token-styles"; +import type { KeyedLine, KeyedToken } from "@/utils/highlight-cache"; + +// Line height kept in sync with the plain `scrollText` style in +// tool-call-details so a highlighted block lines up with an unhighlighted +// fallback and blank lines keep their height. +const CODE_LINE_HEIGHT = 18; +const ZERO_WIDTH = "​"; + +interface HighlightedLinesProps { + lines: KeyedLine[]; + // 1-based line number of the first line; when set, a line-number gutter is + // rendered (used by Read, which carries a server-normalized offset). + startLine?: number; +} + +function ContentLine({ line }: { line: KeyedLine }) { + return ( + + {line.tokens.length === 0 + ? ZERO_WIDTH + : line.tokens.map(({ key, token }: KeyedToken) => ( + + {token.text} + + ))} + + ); +} + +const GutteredLine = React.memo(function GutteredLine({ + line, + lineNumber, + digits, +}: { + line: KeyedLine; + lineNumber: number; + digits: number; +}) { + return ( + + {String(lineNumber).padStart(digits)} + + + ); +}); + +// Renders pre-tokenized lines (from the shared highlight cache), optionally with +// a line-number gutter. Callers decide whether to highlight at all, so the +// expensive size-cap / unsupported-language fallback stays a single plain Text. +export function HighlightedLines({ lines, startLine }: HighlightedLinesProps) { + if (startLine === undefined) { + return ( + + {lines.map((line) => ( + + ))} + + ); + } + + const lastLineNumber = startLine + lines.length - 1; + const digits = Math.max(2, String(lastLineNumber).length); + return ( + + {lines.map((line, index) => ( + + ))} + + ); +} + +const styles = StyleSheet.create((theme) => ({ + row: { + flexDirection: "row", + minHeight: CODE_LINE_HEIGHT, + }, + gutterText: { + fontFamily: Fonts.mono, + fontSize: theme.fontSize.code, + lineHeight: CODE_LINE_HEIGHT, + color: theme.colors.foregroundMuted, + opacity: 0.6, + userSelect: "none", + flexShrink: 0, + }, + lineText: { + fontFamily: Fonts.mono, + fontSize: theme.fontSize.code, + color: theme.colors.foreground, + lineHeight: CODE_LINE_HEIGHT, + ...(isWeb + ? { + whiteSpace: "pre", + overflowWrap: "normal", + } + : null), + }, +})); diff --git a/packages/app/src/components/tool-call-details.tsx b/packages/app/src/components/tool-call-details.tsx index 203232039..e4834e3a3 100644 --- a/packages/app/src/components/tool-call-details.tsx +++ b/packages/app/src/components/tool-call-details.tsx @@ -11,9 +11,12 @@ import { StyleSheet } from "react-native-unistyles"; import { Fonts } from "@/constants/theme"; import type { ToolCallDetail } from "@getpaseo/protocol/agent-types"; import { buildLineDiff, parseUnifiedDiff, type DiffLine } from "@/utils/tool-call-parsers"; +import { highlightDiffLines } from "@/utils/diff-highlight"; import { hasMeaningfulToolCallDetail } from "@/utils/tool-call-detail-state"; import { useWebScrollbarStyle } from "@/hooks/use-web-scrollbar-style"; import { inlineUnistylesStyle } from "@/styles/unistyles-inline-style"; +import { extensionFromPath, highlightToKeyedLines } from "@/utils/highlight-cache"; +import { HighlightedLines } from "./highlighted-content"; import { DiffViewer } from "./diff-viewer"; import { getCodeInsets } from "./code-insets"; import { isWeb } from "@/constants/platform"; @@ -143,10 +146,10 @@ function useDetailStyles( function useDiffLines(detail: ToolCallDetail | undefined): DiffLine[] | undefined { return useMemo(() => { if (!detail || detail.type !== "edit") return undefined; - if (detail.unifiedDiff) { - return parseUnifiedDiff(detail.unifiedDiff); - } - return buildLineDiff(detail.oldString ?? "", detail.newString ?? ""); + const diffLines = detail.unifiedDiff + ? parseUnifiedDiff(detail.unifiedDiff) + : buildLineDiff(detail.oldString ?? "", detail.newString ?? ""); + return highlightDiffLines(diffLines, detail.filePath); }, [detail]); } @@ -431,9 +434,22 @@ interface ScrollableContentProps { content: string; ds: DetailStyles; wrapInSectionFill?: boolean; + // Drives syntax highlighting (extension only) and, with startLine, a gutter. + filePath?: string | null; + startLine?: number; } -function ScrollableTextSection({ content, ds, wrapInSectionFill = true }: ScrollableContentProps) { +function ScrollableTextSection({ + content, + ds, + wrapInSectionFill = true, + filePath, + startLine, +}: ScrollableContentProps) { + const keyedLines = useMemo( + () => (filePath ? highlightToKeyedLines(content, extensionFromPath(filePath)) : null), + [content, filePath], + ); const body = ( - - {content} - + {keyedLines ? ( + + ) : ( + + {content} + + )} ); @@ -669,14 +689,27 @@ function buildDetailSections( return [ {detail.content ? ( - + ) : null} , ]; } if (detail.type === "read") { if (!detail.content) return []; - return []; + return [ + , + ]; } if (detail.type === "search") { return buildSearchSections(detail, ds); diff --git a/packages/app/src/utils/diff-highlight.test.ts b/packages/app/src/utils/diff-highlight.test.ts new file mode 100644 index 000000000..cc3dbd35b --- /dev/null +++ b/packages/app/src/utils/diff-highlight.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { highlightDiffLines } from "./diff-highlight"; +import { buildLineDiff, parseUnifiedDiff, type DiffLine } from "./tool-call-parsers"; + +function joinTokens(line: DiffLine): string { + return (line.tokens ?? []).map((token) => token.text).join(""); +} + +describe("highlightDiffLines", () => { + it("attaches tokens to add/remove/context lines for a supported language", () => { + const diff = buildLineDiff("const a = 1;\nconst b = 2;", "const a = 1;\nconst b = 3;"); + const result = highlightDiffLines(diff, "/repo/src/index.ts"); + + const context = result.find((line) => line.type === "context"); + const remove = result.find((line) => line.type === "remove"); + const add = result.find((line) => line.type === "add"); + + expect(context?.tokens).toBeDefined(); + expect(remove?.tokens).toBeDefined(); + expect(add?.tokens).toBeDefined(); + // Tokens reconstruct the code with the diff marker excluded. + expect(joinTokens(context as DiffLine)).toBe("const a = 1;"); + expect(joinTokens(remove as DiffLine)).toBe("const b = 2;"); + expect(joinTokens(add as DiffLine)).toBe("const b = 3;"); + // A keyword token is recognised (proves real highlighting, not pass-through). + expect(add?.tokens?.some((token) => token.style === "keyword")).toBe(true); + }); + + it("highlights a Codex-style unified diff with bare @@ headers (no line ranges)", () => { + const unified = ["@@", "-const x = 1;", "+const x = 2;", " const y = 3;"].join("\n"); + const result = highlightDiffLines(parseUnifiedDiff(unified), "/repo/a.ts"); + + const remove = result.find((line) => line.type === "remove"); + const add = result.find((line) => line.type === "add"); + const context = result.find((line) => line.type === "context"); + expect(joinTokens(remove as DiffLine)).toBe("const x = 1;"); + expect(joinTokens(add as DiffLine)).toBe("const x = 2;"); + expect(joinTokens(context as DiffLine)).toBe("const y = 3;"); + expect(result.find((line) => line.type === "header")?.tokens).toBeUndefined(); + }); + + it("returns lines unchanged for an unsupported language", () => { + const diff = buildLineDiff("a", "b"); + const result = highlightDiffLines(diff, "/repo/notes.unknownext"); + expect(result).toBe(diff); + expect(result.every((line) => line.tokens === undefined)).toBe(true); + }); + + it("returns lines unchanged when there is no file path", () => { + const diff = buildLineDiff("a", "b"); + expect(highlightDiffLines(diff, undefined)).toBe(diff); + }); +}); diff --git a/packages/app/src/utils/diff-highlight.ts b/packages/app/src/utils/diff-highlight.ts new file mode 100644 index 000000000..cba57786d --- /dev/null +++ b/packages/app/src/utils/diff-highlight.ts @@ -0,0 +1,91 @@ +import { isLanguageSupported, type HighlightToken } from "@getpaseo/highlight"; +import { extensionFromPath, tokenizeToLines } from "@/utils/highlight-cache"; +import type { DiffLine } from "@/utils/tool-call-parsers"; + +// The leading diff marker glyph for a line. The code on the line is the content +// with this prefix removed; we render the glyph separately so token colors +// apply only to the code. +export function diffLinePrefix(line: DiffLine): string { + switch (line.type) { + case "add": + return "+"; + case "remove": + return "-"; + case "context": + return " "; + default: + return ""; + } +} + +function diffLineCode(line: DiffLine): string { + const { content, type } = line; + if (type === "add" || type === "remove") { + return content.startsWith(type === "add" ? "+" : "-") ? content.slice(1) : content; + } + if (type === "context") { + return content.startsWith(" ") ? content.slice(1) : content; + } + return content; +} + +// Attach syntax-highlight tokens to each diff line. Language comes from the file +// path (extension only). We reconstruct the old and new document text from the +// diff lines by position — counting context/remove into "old" and context/add +// into "new" — so this works regardless of whether the source diff carried real +// `@@ -n,m +n,m @@` line ranges (Codex emits bare `@@`). Each document is +// highlighted as a whole so the parser has cross-line context (multi-line +// strings, template literals, comments). Returns the input unchanged when the +// language is unsupported or the content exceeds the highlighter size cap. +export function highlightDiffLines( + diffLines: DiffLine[], + filePath: string | null | undefined, +): DiffLine[] { + const ext = extensionFromPath(filePath); + // Gate on real grammar support: an unsupported language would tokenize to a + // single style-less token per line, which would shadow the word-level change + // segments the diff already computes. Better to keep those. + if (!ext || diffLines.length === 0 || !isLanguageSupported(`x.${ext}`)) { + return diffLines; + } + + const oldCode: string[] = []; + const newCode: string[] = []; + const positions = diffLines.map((line) => { + const code = diffLineCode(line); + if (line.type === "context") { + const position = { oldIndex: oldCode.length, newIndex: newCode.length }; + oldCode.push(code); + newCode.push(code); + return position; + } + if (line.type === "remove") { + const position = { oldIndex: oldCode.length, newIndex: -1 }; + oldCode.push(code); + return position; + } + if (line.type === "add") { + const position = { oldIndex: -1, newIndex: newCode.length }; + newCode.push(code); + return position; + } + return { oldIndex: -1, newIndex: -1 }; + }); + + const oldTokens = oldCode.length > 0 ? tokenizeToLines(oldCode.join("\n"), ext) : null; + const newTokens = newCode.length > 0 ? tokenizeToLines(newCode.join("\n"), ext) : null; + if (!oldTokens && !newTokens) { + return diffLines; + } + + return diffLines.map((line, index) => { + const { oldIndex, newIndex } = positions[index]; + let tokens: HighlightToken[] | undefined; + if ((line.type === "add" || line.type === "context") && newTokens) { + tokens = newTokens[newIndex]; + } else if (line.type === "remove" && oldTokens) { + tokens = oldTokens[oldIndex]; + } + return tokens ? { ...line, tokens } : line; + }); +} diff --git a/packages/app/src/utils/highlight-cache.test.ts b/packages/app/src/utils/highlight-cache.test.ts new file mode 100644 index 000000000..acf140053 --- /dev/null +++ b/packages/app/src/utils/highlight-cache.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { + extensionFromPath, + highlightToKeyedLines, + MAX_HIGHLIGHT_CHARS, + tokenizeToLines, +} from "./highlight-cache"; + +describe("extensionFromPath", () => { + it("extracts a lowercased extension regardless of absolute/relative path", () => { + expect(extensionFromPath("/repo/src/Index.TS")).toBe("ts"); + expect(extensionFromPath("src/index.ts")).toBe("ts"); + expect(extensionFromPath("a.b/c.tsx")).toBe("tsx"); + }); + + it("returns null for paths without a usable extension", () => { + expect(extensionFromPath(null)).toBeNull(); + expect(extensionFromPath(undefined)).toBeNull(); + expect(extensionFromPath("Makefile")).toBeNull(); + expect(extensionFromPath(".gitignore")).toBeNull(); + expect(extensionFromPath("trailingdot.")).toBeNull(); + }); +}); + +describe("tokenizeToLines", () => { + it("returns one token array per line for a supported language", () => { + const lines = tokenizeToLines("const a = 1;\nconst b = 2;", "ts"); + expect(lines).not.toBeNull(); + expect(lines).toHaveLength(2); + expect(lines?.[0].some((token) => token.style === "keyword")).toBe(true); + }); + + it("returns null when there is no extension", () => { + expect(tokenizeToLines("whatever", null)).toBeNull(); + }); + + it("falls back to style-less per-line tokens for an unknown extension", () => { + const lines = tokenizeToLines("line one\nline two", "unknownext"); + expect(lines).toHaveLength(2); + expect(lines?.[0]).toEqual([{ text: "line one", style: null }]); + }); + + it("returns null above the size cap so callers fall back to plain text", () => { + const huge = "x".repeat(MAX_HIGHLIGHT_CHARS + 1); + expect(tokenizeToLines(huge, "ts")).toBeNull(); + }); + + it("serves a cached result on repeat calls (identity-stable)", () => { + const first = tokenizeToLines("const cached = true;", "ts"); + const second = tokenizeToLines("const cached = true;", "ts"); + expect(first).toBe(second); + }); +}); + +describe("highlightToKeyedLines", () => { + it("produces stable keys for lines and tokens", () => { + const keyed = highlightToKeyedLines("const a = 1;", "ts"); + expect(keyed?.[0].key).toBe("line-0"); + expect(keyed?.[0].tokens[0].key).toBe("0-0"); + }); + + it("returns null when highlighting is unavailable", () => { + expect(highlightToKeyedLines("text", null)).toBeNull(); + }); +}); diff --git a/packages/app/src/utils/highlight-cache.ts b/packages/app/src/utils/highlight-cache.ts new file mode 100644 index 000000000..548d6e6fb --- /dev/null +++ b/packages/app/src/utils/highlight-cache.ts @@ -0,0 +1,89 @@ +import { highlightCode, type HighlightToken } from "@getpaseo/highlight"; + +// Shared, theme-independent tokenization + cache for syntax highlighting. +// Used by markdown code blocks, file preview, and tool-call detail blocks +// (Edit diff / Write / Read). Colors are applied at render time, so the cache +// key is just (extension, code) and one entry serves both light and dark. + +export interface KeyedToken { + key: string; + token: HighlightToken; +} + +export interface KeyedLine { + key: string; + tokens: KeyedToken[]; +} + +// Above this, highlighting a whole document on the main thread risks a visible +// stall when a large Read/Write block is expanded. Callers fall back to plain +// monospace text. Generous enough to cover the vast majority of real blocks. +export const MAX_HIGHLIGHT_CHARS = 100_000; + +class LRUCache { + private readonly map = new Map(); + constructor(private readonly max: number) {} + + get(key: K): V | undefined { + const value = this.map.get(key); + if (value === undefined) return undefined; + this.map.delete(key); + this.map.set(key, value); + return value; + } + + set(key: K, value: V): void { + if (this.map.has(key)) this.map.delete(key); + else if (this.map.size >= this.max) { + const oldest = this.map.keys().next().value; + if (oldest !== undefined) this.map.delete(oldest); + } + this.map.set(key, value); + } +} + +const tokenizationCache = new LRUCache(200); + +// Tokenize `code` to per-line tokens, cached. Returns null when the language is +// unsupported, the input is over the size cap, or parsing throws — callers then +// render plain text. +export function tokenizeToLines(code: string, ext: string | null): HighlightToken[][] | null { + if (!ext) return null; + if (code.length > MAX_HIGHLIGHT_CHARS) return null; + const cacheKey = `${ext}:${code}`; + const cached = tokenizationCache.get(cacheKey); + if (cached) return cached; + let lines: HighlightToken[][]; + try { + lines = highlightCode(code, `x.${ext}`); + } catch { + return null; + } + tokenizationCache.set(cacheKey, lines); + return lines; +} + +function toKeyedLine(tokens: HighlightToken[], lineIndex: number): KeyedLine { + return { + key: `line-${lineIndex}`, + tokens: tokens.map((token, tokenIndex) => ({ + key: `${lineIndex}-${tokenIndex}`, + token, + })), + }; +} + +export function highlightToKeyedLines(code: string, ext: string | null): KeyedLine[] | null { + const lines = tokenizeToLines(code, ext); + return lines ? lines.map(toKeyedLine) : null; +} + +// Extension for grammar selection from a file path. We only need the suffix — +// absolute vs relative paths are equivalent here. +export function extensionFromPath(filePath: string | null | undefined): string | null { + if (!filePath) return null; + const name = filePath.split(/[\\/]/).pop() ?? filePath; + const dot = name.lastIndexOf("."); + if (dot <= 0 || dot === name.length - 1) return null; + return name.slice(dot + 1).toLowerCase(); +} diff --git a/packages/app/src/utils/tool-call-parsers.ts b/packages/app/src/utils/tool-call-parsers.ts index 5ed71f0d7..1e89e2b30 100644 --- a/packages/app/src/utils/tool-call-parsers.ts +++ b/packages/app/src/utils/tool-call-parsers.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import type { HighlightToken } from "@getpaseo/highlight"; export interface DiffSegment { text: string; @@ -9,6 +10,9 @@ export interface DiffLine { type: "add" | "remove" | "context" | "header"; content: string; segments?: DiffSegment[]; + // Syntax-highlight tokens for the code on this line (prefix char excluded), + // attached by highlightDiffLines when the file's language is supported. + tokens?: HighlightToken[]; } function splitIntoLines(text: string): string[] { diff --git a/packages/server/src/server/agent/providers/tool-call-detail-primitives.ts b/packages/server/src/server/agent/providers/tool-call-detail-primitives.ts index 54ea8adf8..98bf594bb 100644 --- a/packages/server/src/server/agent/providers/tool-call-detail-primitives.ts +++ b/packages/server/src/server/agent/providers/tool-call-detail-primitives.ts @@ -5,6 +5,7 @@ import { extractCodexShellOutput, flattenReadContent as flattenToolReadContent, nonEmptyString, + stripReadLineNumberGutter, truncateDiffText, } from "./tool-call-mapper-utils.js"; @@ -776,11 +777,15 @@ export function toReadToolDetail( return undefined; } + const stripped = output?.content ? stripReadLineNumberGutter(output.content) : undefined; + const content = stripped?.content ?? output?.content; + const offset = input?.offset ?? stripped?.startLine; + return { type: "read", filePath, - ...(output?.content ? { content: output.content } : {}), - ...(input?.offset !== undefined ? { offset: input.offset } : {}), + ...(content ? { content } : {}), + ...(offset !== undefined ? { offset } : {}), ...(input?.limit !== undefined ? { limit: input.limit } : {}), }; } diff --git a/packages/server/src/server/agent/providers/tool-call-mapper-utils.ts b/packages/server/src/server/agent/providers/tool-call-mapper-utils.ts index 042e757ca..281a2bcf3 100644 --- a/packages/server/src/server/agent/providers/tool-call-mapper-utils.ts +++ b/packages/server/src/server/agent/providers/tool-call-mapper-utils.ts @@ -148,6 +148,81 @@ export function flattenReadContent( ); } +export interface StrippedReadContent { + content: string; + startLine?: number; +} + +// Claude's Read tool returns `cat -n`-style content: each line prefixed with a +// right-aligned line number and a tab (`␣␣␣1\timport ...`). Other providers +// return raw source. We strip the gutter here so `read.content` is uniformly +// raw source across providers, and surface the first line number as `offset` +// so the client can rebuild the gutter itself. Guarded tightly (first line must +// match, near-total match ratio, strictly sequential numbering) so real source +// is never mistaken for a gutter. +const READ_GUTTER_LINE = /^\s*(\d+)\t(.*)$/; + +export function stripReadLineNumberGutter( + content: string | undefined, +): StrippedReadContent | undefined { + const text = nonEmptyString(content); + if (!text) { + return undefined; + } + + const lines = text.replace(/\r\n/g, "\n").split("\n"); + const stripped: string[] = []; + let nonEmpty = 0; + let matched = 0; + let startLine: number | undefined; + let prevNumber: number | undefined; + let sequential = true; + let firstNonEmptyMatched = false; + let sawNonEmpty = false; + + for (const line of lines) { + if (line.length === 0) { + stripped.push(line); + continue; + } + nonEmpty += 1; + const match = line.match(READ_GUTTER_LINE); + if (!match) { + if (!sawNonEmpty) { + return undefined; + } + stripped.push(line); + sawNonEmpty = true; + continue; + } + if (!sawNonEmpty) { + firstNonEmptyMatched = true; + } + sawNonEmpty = true; + matched += 1; + const lineNumber = Number.parseInt(match[1], 10); + if (startLine === undefined) { + startLine = lineNumber; + } + if (prevNumber !== undefined && lineNumber !== prevNumber + 1) { + sequential = false; + } + prevNumber = lineNumber; + stripped.push(match[2]); + } + + if (!firstNonEmptyMatched || !sequential || nonEmpty === 0) { + return undefined; + } + // Sequential numbering from the first line is already a strong signal; the + // ratio only rejects source that has a couple of coincidental matches. + if (matched / nonEmpty < 0.5) { + return undefined; + } + + return { content: stripped.join("\n"), startLine }; +} + export function truncateDiffText( text: string | undefined, maxChars: number = 12_000, diff --git a/packages/server/src/server/agent/providers/tool-call-read-gutter.test.ts b/packages/server/src/server/agent/providers/tool-call-read-gutter.test.ts new file mode 100644 index 000000000..d2795b0a9 --- /dev/null +++ b/packages/server/src/server/agent/providers/tool-call-read-gutter.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; + +import { toReadToolDetail } from "./tool-call-detail-primitives.js"; +import { stripReadLineNumberGutter } from "./tool-call-mapper-utils.js"; + +describe("stripReadLineNumberGutter", () => { + it("strips a cat -n gutter and reports the first line number", () => { + const result = stripReadLineNumberGutter( + ["1\timport a from 'a';", "2\t", "3\tconst x = 1;"].join("\n"), + ); + expect(result).toEqual({ + content: ["import a from 'a';", "", "const x = 1;"].join("\n"), + startLine: 1, + }); + }); + + it("handles right-aligned numbers and a non-1 start offset", () => { + const result = stripReadLineNumberGutter([" 41\tconst y = 2;", " 42\treturn y;"].join("\n")); + expect(result).toEqual({ + content: ["const y = 2;", "return y;"].join("\n"), + startLine: 41, + }); + }); + + it("keeps a trailing non-gutter notice but still strips the body", () => { + const result = stripReadLineNumberGutter( + ["1\tline one", "2\tline two", "truncated"].join("\n"), + ); + expect(result?.content).toBe( + ["line one", "line two", "truncated"].join("\n"), + ); + expect(result?.startLine).toBe(1); + }); + + it("does not strip raw source without a gutter", () => { + expect(stripReadLineNumberGutter("import a from 'a';\nconst x = 1;")).toBeUndefined(); + }); + + it("does not strip when numbering is not sequential", () => { + expect(stripReadLineNumberGutter("1\tfoo\n5\tbar")).toBeUndefined(); + }); + + it("does not strip when the first non-empty line lacks a gutter", () => { + expect(stripReadLineNumberGutter("header line\n1\tfoo\n2\tbar")).toBeUndefined(); + }); + + it("returns undefined for empty content", () => { + expect(stripReadLineNumberGutter(undefined)).toBeUndefined(); + expect(stripReadLineNumberGutter("")).toBeUndefined(); + }); +}); + +describe("toReadToolDetail gutter normalization", () => { + it("strips the gutter from output content and derives offset", () => { + const detail = toReadToolDetail( + { filePath: "/repo/src/index.ts" }, + { content: "10\tconst a = 1;\n11\tconst b = 2;" }, + ); + expect(detail).toEqual({ + type: "read", + filePath: "/repo/src/index.ts", + content: "const a = 1;\nconst b = 2;", + offset: 10, + }); + }); + + it("prefers an explicit input offset over the gutter start", () => { + const detail = toReadToolDetail( + { filePath: "/repo/src/index.ts", offset: 10, limit: 2 }, + { content: "10\tconst a = 1;\n11\tconst b = 2;" }, + ); + expect(detail).toMatchObject({ offset: 10, limit: 2, content: "const a = 1;\nconst b = 2;" }); + }); + + it("leaves raw content untouched", () => { + const detail = toReadToolDetail( + { filePath: "/repo/src/index.ts" }, + { content: "const a = 1;\nconst b = 2;" }, + ); + expect(detail).toEqual({ + type: "read", + filePath: "/repo/src/index.ts", + content: "const a = 1;\nconst b = 2;", + }); + }); +});