Merge branch 'main' of github.com:getpaseo/paseo

packages/app/src/utils/diff-highlight.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from "vitest";
+import { highlightDiffLines } from "./diff-highlight";
+import { buildLineDiff, parseUnifiedDiff, type DiffLine } from "./tool-call-parsers";
+
+function joinTokens(line: DiffLine): string {
+  return (line.tokens ?? []).map((token) => token.text).join("");
+}
+
+describe("highlightDiffLines", () => {
+  it("attaches tokens to add/remove/context lines for a supported language", () => {
+    const diff = buildLineDiff("const a = 1;\nconst b = 2;", "const a = 1;\nconst b = 3;");
+    const result = highlightDiffLines(diff, "/repo/src/index.ts");
+
+    const context = result.find((line) => line.type === "context");
+    const remove = result.find((line) => line.type === "remove");
+    const add = result.find((line) => line.type === "add");
+
+    expect(context?.tokens).toBeDefined();
+    expect(remove?.tokens).toBeDefined();
+    expect(add?.tokens).toBeDefined();
+    // Tokens reconstruct the code with the diff marker excluded.
+    expect(joinTokens(context as DiffLine)).toBe("const a = 1;");
+    expect(joinTokens(remove as DiffLine)).toBe("const b = 2;");
+    expect(joinTokens(add as DiffLine)).toBe("const b = 3;");
+    // A keyword token is recognised (proves real highlighting, not pass-through).
+    expect(add?.tokens?.some((token) => token.style === "keyword")).toBe(true);
+  });
+
+  it("highlights a Codex-style unified diff with bare @@ headers (no line ranges)", () => {
+    const unified = ["@@", "-const x = 1;", "+const x = 2;", " const y = 3;"].join("\n");
+    const result = highlightDiffLines(parseUnifiedDiff(unified), "/repo/a.ts");
+
+    const remove = result.find((line) => line.type === "remove");
+    const add = result.find((line) => line.type === "add");
+    const context = result.find((line) => line.type === "context");
+    expect(joinTokens(remove as DiffLine)).toBe("const x = 1;");
+    expect(joinTokens(add as DiffLine)).toBe("const x = 2;");
+    expect(joinTokens(context as DiffLine)).toBe("const y = 3;");
+    expect(result.find((line) => line.type === "header")?.tokens).toBeUndefined();
+  });
+
+  it("returns lines unchanged for an unsupported language", () => {
+    const diff = buildLineDiff("a", "b");
+    const result = highlightDiffLines(diff, "/repo/notes.unknownext");
+    expect(result).toBe(diff);
+    expect(result.every((line) => line.tokens === undefined)).toBe(true);
+  });
+
+  it("returns lines unchanged when there is no file path", () => {
+    const diff = buildLineDiff("a", "b");
+    expect(highlightDiffLines(diff, undefined)).toBe(diff);
+  });
+});

packages/app/src/utils/diff-highlight.ts

@@ -0,0 +1,91 @@
+import { isLanguageSupported, type HighlightToken } from "@getpaseo/highlight";
+import { extensionFromPath, tokenizeToLines } from "@/utils/highlight-cache";
+import type { DiffLine } from "@/utils/tool-call-parsers";
+
+// The leading diff marker glyph for a line. The code on the line is the content
+// with this prefix removed; we render the glyph separately so token colors
+// apply only to the code.
+export function diffLinePrefix(line: DiffLine): string {
+  switch (line.type) {
+    case "add":
+      return "+";
+    case "remove":
+      return "-";
+    case "context":
+      return " ";
+    default:
+      return "";
+  }
+}
+
+function diffLineCode(line: DiffLine): string {
+  const { content, type } = line;
+  if (type === "add" || type === "remove") {
+    return content.startsWith(type === "add" ? "+" : "-") ? content.slice(1) : content;
+  }
+  if (type === "context") {
+    return content.startsWith(" ") ? content.slice(1) : content;
+  }
+  return content;
+}
+
+// Attach syntax-highlight tokens to each diff line. Language comes from the file
+// path (extension only). We reconstruct the old and new document text from the
+// diff lines by position — counting context/remove into "old" and context/add
+// into "new" — so this works regardless of whether the source diff carried real
+// `@@ -n,m +n,m @@` line ranges (Codex emits bare `@@`). Each document is
+// highlighted as a whole so the parser has cross-line context (multi-line
+// strings, template literals, comments). Returns the input unchanged when the
+// language is unsupported or the content exceeds the highlighter size cap.
+export function highlightDiffLines(
+  diffLines: DiffLine[],
+  filePath: string | null | undefined,
+): DiffLine[] {
+  const ext = extensionFromPath(filePath);
+  // Gate on real grammar support: an unsupported language would tokenize to a
+  // single style-less token per line, which would shadow the word-level change
+  // segments the diff already computes. Better to keep those.
+  if (!ext || diffLines.length === 0 || !isLanguageSupported(`x.${ext}`)) {
+    return diffLines;
+  }
+
+  const oldCode: string[] = [];
+  const newCode: string[] = [];
+  const positions = diffLines.map((line) => {
+    const code = diffLineCode(line);
+    if (line.type === "context") {
+      const position = { oldIndex: oldCode.length, newIndex: newCode.length };
+      oldCode.push(code);
+      newCode.push(code);
+      return position;
+    }
+    if (line.type === "remove") {
+      const position = { oldIndex: oldCode.length, newIndex: -1 };
+      oldCode.push(code);
+      return position;
+    }
+    if (line.type === "add") {
+      const position = { oldIndex: -1, newIndex: newCode.length };
+      newCode.push(code);
+      return position;
+    }
+    return { oldIndex: -1, newIndex: -1 };
+  });
+
+  const oldTokens = oldCode.length > 0 ? tokenizeToLines(oldCode.join("\n"), ext) : null;
+  const newTokens = newCode.length > 0 ? tokenizeToLines(newCode.join("\n"), ext) : null;
+  if (!oldTokens && !newTokens) {
+    return diffLines;
+  }
+
+  return diffLines.map((line, index) => {
+    const { oldIndex, newIndex } = positions[index];
+    let tokens: HighlightToken[] | undefined;
+    if ((line.type === "add" || line.type === "context") && newTokens) {
+      tokens = newTokens[newIndex];
+    } else if (line.type === "remove" && oldTokens) {
+      tokens = oldTokens[oldIndex];
+    }
+    return tokens ? { ...line, tokens } : line;
+  });
+}

packages/app/src/utils/highlight-cache.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "vitest";
+import {
+  extensionFromPath,
+  highlightToKeyedLines,
+  MAX_HIGHLIGHT_CHARS,
+  tokenizeToLines,
+} from "./highlight-cache";
+
+describe("extensionFromPath", () => {
+  it("extracts a lowercased extension regardless of absolute/relative path", () => {
+    expect(extensionFromPath("/repo/src/Index.TS")).toBe("ts");
+    expect(extensionFromPath("src/index.ts")).toBe("ts");
+    expect(extensionFromPath("a.b/c.tsx")).toBe("tsx");
+  });
+
+  it("returns null for paths without a usable extension", () => {
+    expect(extensionFromPath(null)).toBeNull();
+    expect(extensionFromPath(undefined)).toBeNull();
+    expect(extensionFromPath("Makefile")).toBeNull();
+    expect(extensionFromPath(".gitignore")).toBeNull();
+    expect(extensionFromPath("trailingdot.")).toBeNull();
+  });
+});
+
+describe("tokenizeToLines", () => {
+  it("returns one token array per line for a supported language", () => {
+    const lines = tokenizeToLines("const a = 1;\nconst b = 2;", "ts");
+    expect(lines).not.toBeNull();
+    expect(lines).toHaveLength(2);
+    expect(lines?.[0].some((token) => token.style === "keyword")).toBe(true);
+  });
+
+  it("returns null when there is no extension", () => {
+    expect(tokenizeToLines("whatever", null)).toBeNull();
+  });
+
+  it("falls back to style-less per-line tokens for an unknown extension", () => {
+    const lines = tokenizeToLines("line one\nline two", "unknownext");
+    expect(lines).toHaveLength(2);
+    expect(lines?.[0]).toEqual([{ text: "line one", style: null }]);
+  });
+
+  it("returns null above the size cap so callers fall back to plain text", () => {
+    const huge = "x".repeat(MAX_HIGHLIGHT_CHARS + 1);
+    expect(tokenizeToLines(huge, "ts")).toBeNull();
+  });
+
+  it("serves a cached result on repeat calls (identity-stable)", () => {
+    const first = tokenizeToLines("const cached = true;", "ts");
+    const second = tokenizeToLines("const cached = true;", "ts");
+    expect(first).toBe(second);
+  });
+});
+
+describe("highlightToKeyedLines", () => {
+  it("produces stable keys for lines and tokens", () => {
+    const keyed = highlightToKeyedLines("const a = 1;", "ts");
+    expect(keyed?.[0].key).toBe("line-0");
+    expect(keyed?.[0].tokens[0].key).toBe("0-0");
+  });
+
+  it("returns null when highlighting is unavailable", () => {
+    expect(highlightToKeyedLines("text", null)).toBeNull();
+  });
+});

packages/app/src/utils/highlight-cache.ts

@@ -0,0 +1,89 @@
+import { highlightCode, type HighlightToken } from "@getpaseo/highlight";
+
+// Shared, theme-independent tokenization + cache for syntax highlighting.
+// Used by markdown code blocks, file preview, and tool-call detail blocks
+// (Edit diff / Write / Read). Colors are applied at render time, so the cache
+// key is just (extension, code) and one entry serves both light and dark.
+
+export interface KeyedToken {
+  key: string;
+  token: HighlightToken;
+}
+
+export interface KeyedLine {
+  key: string;
+  tokens: KeyedToken[];
+}
+
+// Above this, highlighting a whole document on the main thread risks a visible
+// stall when a large Read/Write block is expanded. Callers fall back to plain
+// monospace text. Generous enough to cover the vast majority of real blocks.
+export const MAX_HIGHLIGHT_CHARS = 100_000;
+
+class LRUCache<K, V> {
+  private readonly map = new Map<K, V>();
+  constructor(private readonly max: number) {}
+
+  get(key: K): V | undefined {
+    const value = this.map.get(key);
+    if (value === undefined) return undefined;
+    this.map.delete(key);
+    this.map.set(key, value);
+    return value;
+  }
+
+  set(key: K, value: V): void {
+    if (this.map.has(key)) this.map.delete(key);
+    else if (this.map.size >= this.max) {
+      const oldest = this.map.keys().next().value;
+      if (oldest !== undefined) this.map.delete(oldest);
+    }
+    this.map.set(key, value);
+  }
+}
+
+const tokenizationCache = new LRUCache<string, HighlightToken[][]>(200);
+
+// Tokenize `code` to per-line tokens, cached. Returns null when the language is
+// unsupported, the input is over the size cap, or parsing throws — callers then
+// render plain text.
+export function tokenizeToLines(code: string, ext: string | null): HighlightToken[][] | null {
+  if (!ext) return null;
+  if (code.length > MAX_HIGHLIGHT_CHARS) return null;
+  const cacheKey = `${ext}:${code}`;
+  const cached = tokenizationCache.get(cacheKey);
+  if (cached) return cached;
+  let lines: HighlightToken[][];
+  try {
+    lines = highlightCode(code, `x.${ext}`);
+  } catch {
+    return null;
+  }
+  tokenizationCache.set(cacheKey, lines);
+  return lines;
+}
+
+function toKeyedLine(tokens: HighlightToken[], lineIndex: number): KeyedLine {
+  return {
+    key: `line-${lineIndex}`,
+    tokens: tokens.map((token, tokenIndex) => ({
+      key: `${lineIndex}-${tokenIndex}`,
+      token,
+    })),
+  };
+}
+
+export function highlightToKeyedLines(code: string, ext: string | null): KeyedLine[] | null {
+  const lines = tokenizeToLines(code, ext);
+  return lines ? lines.map(toKeyedLine) : null;
+}
+
+// Extension for grammar selection from a file path. We only need the suffix —
+// absolute vs relative paths are equivalent here.
+export function extensionFromPath(filePath: string | null | undefined): string | null {
+  if (!filePath) return null;
+  const name = filePath.split(/[\\/]/).pop() ?? filePath;
+  const dot = name.lastIndexOf(".");
+  if (dot <= 0 || dot === name.length - 1) return null;
+  return name.slice(dot + 1).toLowerCase();
+}

packages/app/src/utils/tool-call-parsers.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import type { HighlightToken } from "@getpaseo/highlight";
 
 export interface DiffSegment {
   text: string;
@@ -9,6 +10,9 @@ export interface DiffLine {
   type: "add" | "remove" | "context" | "header";
   content: string;
   segments?: DiffSegment[];
+  // Syntax-highlight tokens for the code on this line (prefix char excluded),
+  // attached by highlightDiffLines when the file's language is supported.
+  tokens?: HighlightToken[];
 }
 
 function splitIntoLines(text: string): string[] {