Syntax-highlight Edit, Write, and Read tool calls (#1218)

Highlighting reuses the existing code-block tokenizer and LRU cache, runs lazily on expand, and falls back to plain text above a size cap so a large Read can't stall the main thread. The diff highlighter reconstructs the old and new file text by position rather than from @@ line ranges, so diffs that arrive without real hunk headers still highlight. Read content is normalized server-side: the cat -n line-number gutter some providers emit is stripped so content is uniformly raw source, with the first line number surfaced as offset for the client to rebuild the gutter.
2026-05-29 14:51:14 +08:00
parent 25d6fc8515
commit 38f0560f9b
12 changed files with 661 additions and 77 deletions
@@ -4,6 +4,8 @@ import { ScrollView as GHScrollView } from "react-native-gesture-handler";
 import { StyleSheet } from "react-native-unistyles";
 import { Fonts } from "@/constants/theme";
 import type { DiffLine } from "@/utils/tool-call-parsers";
+import { diffLinePrefix } from "@/utils/diff-highlight";
+import { syntaxTokenStyleFor } from "@/styles/syntax-token-styles";
 import { useWebScrollbarStyle } from "@/hooks/use-web-scrollbar-style";
 import { inlineUnistylesStyle } from "@/styles/unistyles-inline-style";
 import { getCodeInsets } from "./code-insets";
@@ -40,6 +42,26 @@ function DiffLineRow({ line }: { line: DiffLine }) {
    [line.type],
  );

+  const prefixStyle = React.useMemo(
+    () => [
+      line.type === "add" && styles.addText,
+      line.type === "remove" && styles.removeText,
+      line.type === "context" && styles.contextText,
+    ],
+    [line.type],
+  );
+
+  if (line.tokens) {
+    return (
+      <View style={lineContainerStyle}>
+        <Text style={styles.lineText}>
+          <Text style={prefixStyle}>{diffLinePrefix(line)}</Text>
+          <DiffTokens tokens={line.tokens} />
+        </Text>
+      </View>
+    );
+  }
+
  return (
    <View style={lineContainerStyle}>
      {line.segments ? (
@@ -62,6 +84,22 @@ function DiffLineRow({ line }: { line: DiffLine }) {
  );
 }

+function DiffTokens({ tokens }: { tokens: NonNullable<DiffLine["tokens"]> }) {
+  const keyed = React.useMemo(
+    () => tokens.map((token, index) => ({ key: `${index}-${token.text}`, token })),
+    [tokens],
+  );
+  return (
+    <>
+      {keyed.map(({ key, token }) => (
+        <Text key={key} style={token.style ? syntaxTokenStyleFor(token.style) : undefined}>
+          {token.text}
+        </Text>
+      ))}
+    </>
+  );
+}
+
 function DiffSegment({
  segment,
  lineType,
@@ -4,10 +4,11 @@ import { StyleSheet } from "react-native-unistyles";
 import { MarkdownTextSpan } from "@/components/markdown-text";
 import * as Clipboard from "expo-clipboard";
 import { Check, Copy } from "lucide-react-native";
-import { highlightCode, type HighlightToken } from "@getpaseo/highlight";
+import type { HighlightToken } from "@getpaseo/highlight";
 import { isNative, isWeb } from "@/constants/platform";
 import { useIsCompactFormFactor } from "@/constants/layout";
 import { syntaxTokenStyleFor } from "@/styles/syntax-token-styles";
+import { highlightToKeyedLines, type KeyedLine } from "@/utils/highlight-cache";

 interface HighlightedCodeBlockProps {
  code: string;
@@ -44,34 +45,6 @@ function stripTerminalFenceNewline(code: string): string {
  return code.endsWith("\n") ? code.slice(0, -1) : code;
 }

-// Cross-instance cache for tokenized code blocks. Tokenization is
-// theme-independent (colors are applied at render time), so the key is just
-// (language, code). Bounded by entry count — 200 is generous for a chat
-// transcript, code blocks rarely repeat beyond a handful of distinct shapes.
-class LRUCache<K, V> {
-  private readonly map = new Map<K, V>();
-  constructor(private readonly max: number) {}
-
-  get(key: K): V | undefined {
-    const value = this.map.get(key);
-    if (value === undefined) return undefined;
-    this.map.delete(key);
-    this.map.set(key, value);
-    return value;
-  }
-
-  set(key: K, value: V): void {
-    if (this.map.has(key)) this.map.delete(key);
-    else if (this.map.size >= this.max) {
-      const oldest = this.map.keys().next().value;
-      if (oldest !== undefined) this.map.delete(oldest);
-    }
-    this.map.set(key, value);
-  }
-}
-
-const tokenizationCache = new LRUCache<string, KeyedLine[]>(200);
-
 export const HighlightedCodeBlock = React.memo(function HighlightedCodeBlock({
  code,
  language,
@@ -87,22 +60,10 @@ export const HighlightedCodeBlock = React.memo(function HighlightedCodeBlock({
  );
  const renderedCode = useMemo(() => stripTerminalFenceNewline(code), [code]);

-  const keyedLines = useMemo<KeyedLine[] | null>(() => {
-    const ext = fenceLanguageToExtension(language);
-    if (!ext) return null;
-    const cacheKey = `${ext}:${renderedCode}`;
-    const cached = tokenizationCache.get(cacheKey);
-    if (cached) return cached;
-    let tokenizedLines: HighlightToken[][];
-    try {
-      tokenizedLines = highlightCode(renderedCode, `x.${ext}`);
-    } catch {
-      return null;
-    }
-    const result = tokenizedLines.map(toKeyedLine);
-    tokenizationCache.set(cacheKey, result);
-    return result;
-  }, [renderedCode, language]);
+  const keyedLines = useMemo<KeyedLine[] | null>(
+    () => highlightToKeyedLines(renderedCode, fenceLanguageToExtension(language)),
+    [renderedCode, language],
+  );

  const isCompact = useIsCompactFormFactor();
  const [isHovered, setIsHovered] = useState(false);
@@ -127,26 +88,6 @@ export const HighlightedCodeBlock = React.memo(function HighlightedCodeBlock({
  );
 });

-interface KeyedToken {
-  key: string;
-  token: HighlightToken;
-}
-
-interface KeyedLine {
-  key: string;
-  tokens: KeyedToken[];
-}
-
-function toKeyedLine(tokens: HighlightToken[], lineIndex: number): KeyedLine {
-  return {
-    key: `line-${lineIndex}`,
-    tokens: tokens.map((token, tokenIndex) => ({
-      key: `${lineIndex}-${tokenIndex}`,
-      token,
-    })),
-  };
-}
-
 function renderCodeSegments(keyedLines: KeyedLine[]): React.ReactNode[] {
  const segments: React.ReactNode[] = [];
  for (let lineIndex = 0; lineIndex < keyedLines.length; lineIndex += 1) {
@@ -0,0 +1,104 @@
+import React from "react";
+import { Text, View } from "react-native";
+import { StyleSheet } from "react-native-unistyles";
+import { Fonts } from "@/constants/theme";
+import { isWeb } from "@/constants/platform";
+import { syntaxTokenStyleFor } from "@/styles/syntax-token-styles";
+import type { KeyedLine, KeyedToken } from "@/utils/highlight-cache";
+
+// Line height kept in sync with the plain `scrollText` style in
+// tool-call-details so a highlighted block lines up with an unhighlighted
+// fallback and blank lines keep their height.
+const CODE_LINE_HEIGHT = 18;
+const ZERO_WIDTH = "";
+
+interface HighlightedLinesProps {
+  lines: KeyedLine[];
+  // 1-based line number of the first line; when set, a line-number gutter is
+  // rendered (used by Read, which carries a server-normalized offset).
+  startLine?: number;
+}
+
+function ContentLine({ line }: { line: KeyedLine }) {
+  return (
+    <Text selectable style={styles.lineText}>
+      {line.tokens.length === 0
+        ? ZERO_WIDTH
+        : line.tokens.map(({ key, token }: KeyedToken) => (
+            <Text key={key} style={token.style ? syntaxTokenStyleFor(token.style) : undefined}>
+              {token.text}
+            </Text>
+          ))}
+    </Text>
+  );
+}
+
+const GutteredLine = React.memo(function GutteredLine({
+  line,
+  lineNumber,
+  digits,
+}: {
+  line: KeyedLine;
+  lineNumber: number;
+  digits: number;
+}) {
+  return (
+    <View style={styles.row}>
+      <Text style={styles.gutterText}>{String(lineNumber).padStart(digits)} </Text>
+      <ContentLine line={line} />
+    </View>
+  );
+});
+
+// Renders pre-tokenized lines (from the shared highlight cache), optionally with
+// a line-number gutter. Callers decide whether to highlight at all, so the
+// expensive size-cap / unsupported-language fallback stays a single plain Text.
+export function HighlightedLines({ lines, startLine }: HighlightedLinesProps) {
+  if (startLine === undefined) {
+    return (
+      <View>
+        {lines.map((line) => (
+          <ContentLine key={line.key} line={line} />
+        ))}
+      </View>
+    );
+  }
+
+  const lastLineNumber = startLine + lines.length - 1;
+  const digits = Math.max(2, String(lastLineNumber).length);
+  return (
+    <View>
+      {lines.map((line, index) => (
+        <GutteredLine key={line.key} line={line} lineNumber={startLine + index} digits={digits} />
+      ))}
+    </View>
+  );
+}
+
+const styles = StyleSheet.create((theme) => ({
+  row: {
+    flexDirection: "row",
+    minHeight: CODE_LINE_HEIGHT,
+  },
+  gutterText: {
+    fontFamily: Fonts.mono,
+    fontSize: theme.fontSize.code,
+    lineHeight: CODE_LINE_HEIGHT,
+    color: theme.colors.foregroundMuted,
+    opacity: 0.6,
+    userSelect: "none",
+    flexShrink: 0,
+  },
+  lineText: {
+    fontFamily: Fonts.mono,
+    fontSize: theme.fontSize.code,
+    color: theme.colors.foreground,
+    lineHeight: CODE_LINE_HEIGHT,
+    ...(isWeb
+      ? {
+          whiteSpace: "pre",
+          overflowWrap: "normal",
+        }
+      : null),
+  },
+}));
@@ -11,9 +11,12 @@ import { StyleSheet } from "react-native-unistyles";
 import { Fonts } from "@/constants/theme";
 import type { ToolCallDetail } from "@getpaseo/protocol/agent-types";
 import { buildLineDiff, parseUnifiedDiff, type DiffLine } from "@/utils/tool-call-parsers";
+import { highlightDiffLines } from "@/utils/diff-highlight";
 import { hasMeaningfulToolCallDetail } from "@/utils/tool-call-detail-state";
 import { useWebScrollbarStyle } from "@/hooks/use-web-scrollbar-style";
 import { inlineUnistylesStyle } from "@/styles/unistyles-inline-style";
+import { extensionFromPath, highlightToKeyedLines } from "@/utils/highlight-cache";
+import { HighlightedLines } from "./highlighted-content";
 import { DiffViewer } from "./diff-viewer";
 import { getCodeInsets } from "./code-insets";
 import { isWeb } from "@/constants/platform";
@@ -143,10 +146,10 @@ function useDetailStyles(
 function useDiffLines(detail: ToolCallDetail | undefined): DiffLine[] | undefined {
  return useMemo(() => {
    if (!detail || detail.type !== "edit") return undefined;
-    if (detail.unifiedDiff) {
-      return parseUnifiedDiff(detail.unifiedDiff);
-    }
-    return buildLineDiff(detail.oldString ?? "", detail.newString ?? "");
+    const diffLines = detail.unifiedDiff
+      ? parseUnifiedDiff(detail.unifiedDiff)
+      : buildLineDiff(detail.oldString ?? "", detail.newString ?? "");
+    return highlightDiffLines(diffLines, detail.filePath);
  }, [detail]);
 }

@@ -431,9 +434,22 @@ interface ScrollableContentProps {
  content: string;
  ds: DetailStyles;
  wrapInSectionFill?: boolean;
+  // Drives syntax highlighting (extension only) and, with startLine, a gutter.
+  filePath?: string | null;
+  startLine?: number;
 }

-function ScrollableTextSection({ content, ds, wrapInSectionFill = true }: ScrollableContentProps) {
+function ScrollableTextSection({
+  content,
+  ds,
+  wrapInSectionFill = true,
+  filePath,
+  startLine,
+}: ScrollableContentProps) {
+  const keyedLines = useMemo(
+    () => (filePath ? highlightToKeyedLines(content, extensionFromPath(filePath)) : null),
+    [content, filePath],
+  );
  const body = (
    <ScrollView
      style={ds.scrollAreaFillStyle}
@@ -447,9 +463,13 @@ function ScrollableTextSection({ content, ds, wrapInSectionFill = true }: Scroll
        showsHorizontalScrollIndicator={true}
        style={ds.webScrollbarStyle}
      >
-        <Text selectable style={styles.scrollText}>
-          {content}
-        </Text>
+        {keyedLines ? (
+          <HighlightedLines lines={keyedLines} startLine={startLine} />
+        ) : (
+          <Text selectable style={styles.scrollText}>
+            {content}
+          </Text>
+        )}
      </ScrollView>
    </ScrollView>
  );
@@ -669,14 +689,27 @@ function buildDetailSections(
    return [
      <View key="write" style={ds.sectionFillStyle}>
        {detail.content ? (
-          <ScrollableTextSection content={detail.content} ds={ds} wrapInSectionFill={false} />
+          <ScrollableTextSection
+            content={detail.content}
+            ds={ds}
+            wrapInSectionFill={false}
+            filePath={detail.filePath}
+          />
        ) : null}
      </View>,
    ];
  }
  if (detail.type === "read") {
    if (!detail.content) return [];
-    return [<ScrollableTextSection key="read" content={detail.content} ds={ds} />];
+    return [
+      <ScrollableTextSection
+        key="read"
+        content={detail.content}
+        ds={ds}
+        filePath={detail.filePath}
+        startLine={detail.offset ?? 1}
+      />,
+    ];
  }
  if (detail.type === "search") {
    return buildSearchSections(detail, ds);
@@ -0,0 +1,53 @@
+import { describe, expect, it } from "vitest";
+import { highlightDiffLines } from "./diff-highlight";
+import { buildLineDiff, parseUnifiedDiff, type DiffLine } from "./tool-call-parsers";
+
+function joinTokens(line: DiffLine): string {
+  return (line.tokens ?? []).map((token) => token.text).join("");
+}
+
+describe("highlightDiffLines", () => {
+  it("attaches tokens to add/remove/context lines for a supported language", () => {
+    const diff = buildLineDiff("const a = 1;\nconst b = 2;", "const a = 1;\nconst b = 3;");
+    const result = highlightDiffLines(diff, "/repo/src/index.ts");
+
+    const context = result.find((line) => line.type === "context");
+    const remove = result.find((line) => line.type === "remove");
+    const add = result.find((line) => line.type === "add");
+
+    expect(context?.tokens).toBeDefined();
+    expect(remove?.tokens).toBeDefined();
+    expect(add?.tokens).toBeDefined();
+    // Tokens reconstruct the code with the diff marker excluded.
+    expect(joinTokens(context as DiffLine)).toBe("const a = 1;");
+    expect(joinTokens(remove as DiffLine)).toBe("const b = 2;");
+    expect(joinTokens(add as DiffLine)).toBe("const b = 3;");
+    // A keyword token is recognised (proves real highlighting, not pass-through).
+    expect(add?.tokens?.some((token) => token.style === "keyword")).toBe(true);
+  });
+
+  it("highlights a Codex-style unified diff with bare @@ headers (no line ranges)", () => {
+    const unified = ["@@", "-const x = 1;", "+const x = 2;", " const y = 3;"].join("\n");
+    const result = highlightDiffLines(parseUnifiedDiff(unified), "/repo/a.ts");
+
+    const remove = result.find((line) => line.type === "remove");
+    const add = result.find((line) => line.type === "add");
+    const context = result.find((line) => line.type === "context");
+    expect(joinTokens(remove as DiffLine)).toBe("const x = 1;");
+    expect(joinTokens(add as DiffLine)).toBe("const x = 2;");
+    expect(joinTokens(context as DiffLine)).toBe("const y = 3;");
+    expect(result.find((line) => line.type === "header")?.tokens).toBeUndefined();
+  });
+
+  it("returns lines unchanged for an unsupported language", () => {
+    const diff = buildLineDiff("a", "b");
+    const result = highlightDiffLines(diff, "/repo/notes.unknownext");
+    expect(result).toBe(diff);
+    expect(result.every((line) => line.tokens === undefined)).toBe(true);
+  });
+
+  it("returns lines unchanged when there is no file path", () => {
+    const diff = buildLineDiff("a", "b");
+    expect(highlightDiffLines(diff, undefined)).toBe(diff);
+  });
+});
@@ -0,0 +1,91 @@
+import { isLanguageSupported, type HighlightToken } from "@getpaseo/highlight";
+import { extensionFromPath, tokenizeToLines } from "@/utils/highlight-cache";
+import type { DiffLine } from "@/utils/tool-call-parsers";
+
+// The leading diff marker glyph for a line. The code on the line is the content
+// with this prefix removed; we render the glyph separately so token colors
+// apply only to the code.
+export function diffLinePrefix(line: DiffLine): string {
+  switch (line.type) {
+    case "add":
+      return "+";
+    case "remove":
+      return "-";
+    case "context":
+      return " ";
+    default:
+      return "";
+  }
+}
+
+function diffLineCode(line: DiffLine): string {
+  const { content, type } = line;
+  if (type === "add" || type === "remove") {
+    return content.startsWith(type === "add" ? "+" : "-") ? content.slice(1) : content;
+  }
+  if (type === "context") {
+    return content.startsWith(" ") ? content.slice(1) : content;
+  }
+  return content;
+}
+
+// Attach syntax-highlight tokens to each diff line. Language comes from the file
+// path (extension only). We reconstruct the old and new document text from the
+// diff lines by position — counting context/remove into "old" and context/add
+// into "new" — so this works regardless of whether the source diff carried real
+// `@@ -n,m +n,m @@` line ranges (Codex emits bare `@@`). Each document is
+// highlighted as a whole so the parser has cross-line context (multi-line
+// strings, template literals, comments). Returns the input unchanged when the
+// language is unsupported or the content exceeds the highlighter size cap.
+export function highlightDiffLines(
+  diffLines: DiffLine[],
+  filePath: string | null | undefined,
+): DiffLine[] {
+  const ext = extensionFromPath(filePath);
+  // Gate on real grammar support: an unsupported language would tokenize to a
+  // single style-less token per line, which would shadow the word-level change
+  // segments the diff already computes. Better to keep those.
+  if (!ext || diffLines.length === 0 || !isLanguageSupported(`x.${ext}`)) {
+    return diffLines;
+  }
+
+  const oldCode: string[] = [];
+  const newCode: string[] = [];
+  const positions = diffLines.map((line) => {
+    const code = diffLineCode(line);
+    if (line.type === "context") {
+      const position = { oldIndex: oldCode.length, newIndex: newCode.length };
+      oldCode.push(code);
+      newCode.push(code);
+      return position;
+    }
+    if (line.type === "remove") {
+      const position = { oldIndex: oldCode.length, newIndex: -1 };
+      oldCode.push(code);
+      return position;
+    }
+    if (line.type === "add") {
+      const position = { oldIndex: -1, newIndex: newCode.length };
+      newCode.push(code);
+      return position;
+    }
+    return { oldIndex: -1, newIndex: -1 };
+  });
+
+  const oldTokens = oldCode.length > 0 ? tokenizeToLines(oldCode.join("\n"), ext) : null;
+  const newTokens = newCode.length > 0 ? tokenizeToLines(newCode.join("\n"), ext) : null;
+  if (!oldTokens && !newTokens) {
+    return diffLines;
+  }
+
+  return diffLines.map((line, index) => {
+    const { oldIndex, newIndex } = positions[index];
+    let tokens: HighlightToken[] | undefined;
+    if ((line.type === "add" || line.type === "context") && newTokens) {
+      tokens = newTokens[newIndex];
+    } else if (line.type === "remove" && oldTokens) {
+      tokens = oldTokens[oldIndex];
+    }
+    return tokens ? { ...line, tokens } : line;
+  });
+}
@@ -0,0 +1,65 @@
+import { describe, expect, it } from "vitest";
+import {
+  extensionFromPath,
+  highlightToKeyedLines,
+  MAX_HIGHLIGHT_CHARS,
+  tokenizeToLines,
+} from "./highlight-cache";
+
+describe("extensionFromPath", () => {
+  it("extracts a lowercased extension regardless of absolute/relative path", () => {
+    expect(extensionFromPath("/repo/src/Index.TS")).toBe("ts");
+    expect(extensionFromPath("src/index.ts")).toBe("ts");
+    expect(extensionFromPath("a.b/c.tsx")).toBe("tsx");
+  });
+
+  it("returns null for paths without a usable extension", () => {
+    expect(extensionFromPath(null)).toBeNull();
+    expect(extensionFromPath(undefined)).toBeNull();
+    expect(extensionFromPath("Makefile")).toBeNull();
+    expect(extensionFromPath(".gitignore")).toBeNull();
+    expect(extensionFromPath("trailingdot.")).toBeNull();
+  });
+});
+
+describe("tokenizeToLines", () => {
+  it("returns one token array per line for a supported language", () => {
+    const lines = tokenizeToLines("const a = 1;\nconst b = 2;", "ts");
+    expect(lines).not.toBeNull();
+    expect(lines).toHaveLength(2);
+    expect(lines?.[0].some((token) => token.style === "keyword")).toBe(true);
+  });
+
+  it("returns null when there is no extension", () => {
+    expect(tokenizeToLines("whatever", null)).toBeNull();
+  });
+
+  it("falls back to style-less per-line tokens for an unknown extension", () => {
+    const lines = tokenizeToLines("line one\nline two", "unknownext");
+    expect(lines).toHaveLength(2);
+    expect(lines?.[0]).toEqual([{ text: "line one", style: null }]);
+  });
+
+  it("returns null above the size cap so callers fall back to plain text", () => {
+    const huge = "x".repeat(MAX_HIGHLIGHT_CHARS + 1);
+    expect(tokenizeToLines(huge, "ts")).toBeNull();
+  });
+
+  it("serves a cached result on repeat calls (identity-stable)", () => {
+    const first = tokenizeToLines("const cached = true;", "ts");
+    const second = tokenizeToLines("const cached = true;", "ts");
+    expect(first).toBe(second);
+  });
+});
+
+describe("highlightToKeyedLines", () => {
+  it("produces stable keys for lines and tokens", () => {
+    const keyed = highlightToKeyedLines("const a = 1;", "ts");
+    expect(keyed?.[0].key).toBe("line-0");
+    expect(keyed?.[0].tokens[0].key).toBe("0-0");
+  });
+
+  it("returns null when highlighting is unavailable", () => {
+    expect(highlightToKeyedLines("text", null)).toBeNull();
+  });
+});
@@ -0,0 +1,89 @@
+import { highlightCode, type HighlightToken } from "@getpaseo/highlight";
+
+// Shared, theme-independent tokenization + cache for syntax highlighting.
+// Used by markdown code blocks, file preview, and tool-call detail blocks
+// (Edit diff / Write / Read). Colors are applied at render time, so the cache
+// key is just (extension, code) and one entry serves both light and dark.
+
+export interface KeyedToken {
+  key: string;
+  token: HighlightToken;
+}
+
+export interface KeyedLine {
+  key: string;
+  tokens: KeyedToken[];
+}
+
+// Above this, highlighting a whole document on the main thread risks a visible
+// stall when a large Read/Write block is expanded. Callers fall back to plain
+// monospace text. Generous enough to cover the vast majority of real blocks.
+export const MAX_HIGHLIGHT_CHARS = 100_000;
+
+class LRUCache<K, V> {
+  private readonly map = new Map<K, V>();
+  constructor(private readonly max: number) {}
+
+  get(key: K): V | undefined {
+    const value = this.map.get(key);
+    if (value === undefined) return undefined;
+    this.map.delete(key);
+    this.map.set(key, value);
+    return value;
+  }
+
+  set(key: K, value: V): void {
+    if (this.map.has(key)) this.map.delete(key);
+    else if (this.map.size >= this.max) {
+      const oldest = this.map.keys().next().value;
+      if (oldest !== undefined) this.map.delete(oldest);
+    }
+    this.map.set(key, value);
+  }
+}
+
+const tokenizationCache = new LRUCache<string, HighlightToken[][]>(200);
+
+// Tokenize `code` to per-line tokens, cached. Returns null when the language is
+// unsupported, the input is over the size cap, or parsing throws — callers then
+// render plain text.
+export function tokenizeToLines(code: string, ext: string | null): HighlightToken[][] | null {
+  if (!ext) return null;
+  if (code.length > MAX_HIGHLIGHT_CHARS) return null;
+  const cacheKey = `${ext}:${code}`;
+  const cached = tokenizationCache.get(cacheKey);
+  if (cached) return cached;
+  let lines: HighlightToken[][];
+  try {
+    lines = highlightCode(code, `x.${ext}`);
+  } catch {
+    return null;
+  }
+  tokenizationCache.set(cacheKey, lines);
+  return lines;
+}
+
+function toKeyedLine(tokens: HighlightToken[], lineIndex: number): KeyedLine {
+  return {
+    key: `line-${lineIndex}`,
+    tokens: tokens.map((token, tokenIndex) => ({
+      key: `${lineIndex}-${tokenIndex}`,
+      token,
+    })),
+  };
+}
+
+export function highlightToKeyedLines(code: string, ext: string | null): KeyedLine[] | null {
+  const lines = tokenizeToLines(code, ext);
+  return lines ? lines.map(toKeyedLine) : null;
+}
+
+// Extension for grammar selection from a file path. We only need the suffix —
+// absolute vs relative paths are equivalent here.
+export function extensionFromPath(filePath: string | null | undefined): string | null {
+  if (!filePath) return null;
+  const name = filePath.split(/[\\/]/).pop() ?? filePath;
+  const dot = name.lastIndexOf(".");
+  if (dot <= 0 || dot === name.length - 1) return null;
+  return name.slice(dot + 1).toLowerCase();
+}
@@ -1,4 +1,5 @@
 import { z } from "zod";
+import type { HighlightToken } from "@getpaseo/highlight";

 export interface DiffSegment {
  text: string;
@@ -9,6 +10,9 @@ export interface DiffLine {
  type: "add" | "remove" | "context" | "header";
  content: string;
  segments?: DiffSegment[];
+  // Syntax-highlight tokens for the code on this line (prefix char excluded),
+  // attached by highlightDiffLines when the file's language is supported.
+  tokens?: HighlightToken[];
 }

 function splitIntoLines(text: string): string[] {
@@ -5,6 +5,7 @@ import {
  extractCodexShellOutput,
  flattenReadContent as flattenToolReadContent,
  nonEmptyString,
+  stripReadLineNumberGutter,
  truncateDiffText,
 } from "./tool-call-mapper-utils.js";

@@ -776,11 +777,15 @@ export function toReadToolDetail(
    return undefined;
  }

+  const stripped = output?.content ? stripReadLineNumberGutter(output.content) : undefined;
+  const content = stripped?.content ?? output?.content;
+  const offset = input?.offset ?? stripped?.startLine;
+
  return {
    type: "read",
    filePath,
-    ...(output?.content ? { content: output.content } : {}),
-    ...(input?.offset !== undefined ? { offset: input.offset } : {}),
+    ...(content ? { content } : {}),
+    ...(offset !== undefined ? { offset } : {}),
    ...(input?.limit !== undefined ? { limit: input.limit } : {}),
  };
 }
@@ -148,6 +148,81 @@ export function flattenReadContent<Chunk extends ReadChunkLike>(
  );
 }

+export interface StrippedReadContent {
+  content: string;
+  startLine?: number;
+}
+
+// Claude's Read tool returns `cat -n`-style content: each line prefixed with a
+// right-aligned line number and a tab (`␣␣␣1\timport ...`). Other providers
+// return raw source. We strip the gutter here so `read.content` is uniformly
+// raw source across providers, and surface the first line number as `offset`
+// so the client can rebuild the gutter itself. Guarded tightly (first line must
+// match, near-total match ratio, strictly sequential numbering) so real source
+// is never mistaken for a gutter.
+const READ_GUTTER_LINE = /^\s*(\d+)\t(.*)$/;
+
+export function stripReadLineNumberGutter(
+  content: string | undefined,
+): StrippedReadContent | undefined {
+  const text = nonEmptyString(content);
+  if (!text) {
+    return undefined;
+  }
+
+  const lines = text.replace(/\r\n/g, "\n").split("\n");
+  const stripped: string[] = [];
+  let nonEmpty = 0;
+  let matched = 0;
+  let startLine: number | undefined;
+  let prevNumber: number | undefined;
+  let sequential = true;
+  let firstNonEmptyMatched = false;
+  let sawNonEmpty = false;
+
+  for (const line of lines) {
+    if (line.length === 0) {
+      stripped.push(line);
+      continue;
+    }
+    nonEmpty += 1;
+    const match = line.match(READ_GUTTER_LINE);
+    if (!match) {
+      if (!sawNonEmpty) {
+        return undefined;
+      }
+      stripped.push(line);
+      sawNonEmpty = true;
+      continue;
+    }
+    if (!sawNonEmpty) {
+      firstNonEmptyMatched = true;
+    }
+    sawNonEmpty = true;
+    matched += 1;
+    const lineNumber = Number.parseInt(match[1], 10);
+    if (startLine === undefined) {
+      startLine = lineNumber;
+    }
+    if (prevNumber !== undefined && lineNumber !== prevNumber + 1) {
+      sequential = false;
+    }
+    prevNumber = lineNumber;
+    stripped.push(match[2]);
+  }
+
+  if (!firstNonEmptyMatched || !sequential || nonEmpty === 0) {
+    return undefined;
+  }
+  // Sequential numbering from the first line is already a strong signal; the
+  // ratio only rejects source that has a couple of coincidental matches.
+  if (matched / nonEmpty < 0.5) {
+    return undefined;
+  }
+
+  return { content: stripped.join("\n"), startLine };
+}
+
 export function truncateDiffText(
  text: string | undefined,
  maxChars: number = 12_000,
@@ -0,0 +1,86 @@
+import { describe, expect, it } from "vitest";
+
+import { toReadToolDetail } from "./tool-call-detail-primitives.js";
+import { stripReadLineNumberGutter } from "./tool-call-mapper-utils.js";
+
+describe("stripReadLineNumberGutter", () => {
+  it("strips a cat -n gutter and reports the first line number", () => {
+    const result = stripReadLineNumberGutter(
+      ["1\timport a from 'a';", "2\t", "3\tconst x = 1;"].join("\n"),
+    );
+    expect(result).toEqual({
+      content: ["import a from 'a';", "", "const x = 1;"].join("\n"),
+      startLine: 1,
+    });
+  });
+
+  it("handles right-aligned numbers and a non-1 start offset", () => {
+    const result = stripReadLineNumberGutter(["  41\tconst y = 2;", "  42\treturn y;"].join("\n"));
+    expect(result).toEqual({
+      content: ["const y = 2;", "return y;"].join("\n"),
+      startLine: 41,
+    });
+  });
+
+  it("keeps a trailing non-gutter notice but still strips the body", () => {
+    const result = stripReadLineNumberGutter(
+      ["1\tline one", "2\tline two", "<system-reminder>truncated</system-reminder>"].join("\n"),
+    );
+    expect(result?.content).toBe(
+      ["line one", "line two", "<system-reminder>truncated</system-reminder>"].join("\n"),
+    );
+    expect(result?.startLine).toBe(1);
+  });
+
+  it("does not strip raw source without a gutter", () => {
+    expect(stripReadLineNumberGutter("import a from 'a';\nconst x = 1;")).toBeUndefined();
+  });
+
+  it("does not strip when numbering is not sequential", () => {
+    expect(stripReadLineNumberGutter("1\tfoo\n5\tbar")).toBeUndefined();
+  });
+
+  it("does not strip when the first non-empty line lacks a gutter", () => {
+    expect(stripReadLineNumberGutter("header line\n1\tfoo\n2\tbar")).toBeUndefined();
+  });
+
+  it("returns undefined for empty content", () => {
+    expect(stripReadLineNumberGutter(undefined)).toBeUndefined();
+    expect(stripReadLineNumberGutter("")).toBeUndefined();
+  });
+});
+
+describe("toReadToolDetail gutter normalization", () => {
+  it("strips the gutter from output content and derives offset", () => {
+    const detail = toReadToolDetail(
+      { filePath: "/repo/src/index.ts" },
+      { content: "10\tconst a = 1;\n11\tconst b = 2;" },
+    );
+    expect(detail).toEqual({
+      type: "read",
+      filePath: "/repo/src/index.ts",
+      content: "const a = 1;\nconst b = 2;",
+      offset: 10,
+    });
+  });
+
+  it("prefers an explicit input offset over the gutter start", () => {
+    const detail = toReadToolDetail(
+      { filePath: "/repo/src/index.ts", offset: 10, limit: 2 },
+      { content: "10\tconst a = 1;\n11\tconst b = 2;" },
+    );
+    expect(detail).toMatchObject({ offset: 10, limit: 2, content: "const a = 1;\nconst b = 2;" });
+  });
+
+  it("leaves raw content untouched", () => {
+    const detail = toReadToolDetail(
+      { filePath: "/repo/src/index.ts" },
+      { content: "const a = 1;\nconst b = 2;" },
+    );
+    expect(detail).toEqual({
+      type: "read",
+      filePath: "/repo/src/index.ts",
+      content: "const a = 1;\nconst b = 2;",
+    });
+  });
+});