Fix Shift+Enter in terminal input modes (#999)

* Fix Shift+Enter terminal input modes

* Advertise terminal input capabilities

packages/server/src/shared/terminal-input-mode.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, it } from "vitest";
+import { TerminalInputModeTracker } from "./terminal-input-mode.js";
+
+describe("TerminalInputModeTracker", () => {
+  it("activates from a pushed Kitty keyboard mode and builds a replay preamble", () => {
+    const tracker = new TerminalInputModeTracker();
+
+    expect(tracker.feed("\x1b[>7u").changed).toBe(true);
+
+    expect(tracker.supportsModifiedEnter()).toBe(true);
+    expect(tracker.getKittyKeyboardFlags()).toBe(7);
+    expect(tracker.getPreamble()).toBe("\x1b[=7;1u");
+  });
+
+  it("tracks split terminal output chunks", () => {
+    const tracker = new TerminalInputModeTracker();
+
+    tracker.feed("\x1b[>");
+    const result = tracker.feed("1u");
+
+    expect(result.changed).toBe(true);
+    expect(tracker.getKittyKeyboardFlags()).toBe(1);
+  });
+
+  it("restores pushed Kitty keyboard modes when the foreground program pops them", () => {
+    const tracker = new TerminalInputModeTracker();
+
+    tracker.feed("\x1b[>1u");
+    tracker.feed("\x1b[>7u");
+    expect(tracker.getKittyKeyboardFlags()).toBe(7);
+
+    expect(tracker.feed("\x1b[<u").changed).toBe(true);
+    expect(tracker.getKittyKeyboardFlags()).toBe(1);
+
+    expect(tracker.feed("\x1b[<u").changed).toBe(true);
+    expect(tracker.supportsModifiedEnter()).toBe(false);
+  });
+
+  it("answers Kitty keyboard mode queries with the current flags", () => {
+    const tracker = new TerminalInputModeTracker();
+    tracker.feed("\x1b[=3;1u");
+
+    expect(tracker.feed("\x1b[?u").responses).toEqual(["\x1b[?3u"]);
+  });
+
+  it("tracks ConPTY Win32 input mode and replays it after snapshots", () => {
+    const tracker = new TerminalInputModeTracker();
+
+    expect(tracker.feed("\x1b[?9001h").changed).toBe(true);
+
+    expect(tracker.getState()).toEqual({
+      kittyKeyboardFlags: 0,
+      win32InputMode: true,
+    });
+    expect(tracker.supportsModifiedEnter()).toBe(true);
+    expect(tracker.getPreamble()).toBe("\x1b[?9001h");
+
+    expect(tracker.feed("\x1b[?9001l").changed).toBe(true);
+    expect(tracker.supportsModifiedEnter()).toBe(false);
+  });
+
+  it("keeps Kitty and Win32 input modes independent", () => {
+    const tracker = new TerminalInputModeTracker();
+
+    tracker.feed("\x1b[>7u\x1b[?9001h");
+
+    expect(tracker.getState()).toEqual({
+      kittyKeyboardFlags: 7,
+      win32InputMode: true,
+    });
+    expect(tracker.getPreamble()).toBe("\x1b[=7;1u\x1b[?9001h");
+  });
+
+  it("ignores encoded key input sequences", () => {
+    const tracker = new TerminalInputModeTracker();
+
+    tracker.feed("\x1b[13;2u");
+
+    expect(tracker.supportsModifiedEnter()).toBe(false);
+  });
+});

packages/server/src/shared/terminal-input-mode.ts

@@ -0,0 +1,194 @@
+export interface TerminalInputModeFeedResult {
+  changed: boolean;
+  responses: string[];
+}
+
+export interface TerminalInputModeState {
+  kittyKeyboardFlags: number;
+  win32InputMode: boolean;
+}
+
+export const DEFAULT_TERMINAL_INPUT_MODE_STATE: TerminalInputModeState = {
+  kittyKeyboardFlags: 0,
+  win32InputMode: false,
+};
+
+const ESC = String.fromCharCode(0x1b);
+const WIN32_INPUT_MODE = 9001;
+const CSI_INPUT_MODE_SEQUENCE = new RegExp(
+  `${ESC}\\[(?:([<>=?]?)([0-9;]*)u|\\?([0-9;]*)([hl]))`,
+  "g",
+);
+const INCOMPLETE_CSI_INPUT_MODE_SEQUENCE = new RegExp(`${ESC}\\[[<>=?]?[0-9;]*$`);
+
+function parseFirstParam(params: string): number | null {
+  const first = params.split(";")[0];
+  if (!first || !/^\d+$/.test(first)) {
+    return null;
+  }
+  return Number(first);
+}
+
+function parseSecondParam(params: string): number | null {
+  const second = params.split(";")[1];
+  if (!second || !/^\d+$/.test(second)) {
+    return null;
+  }
+  return Number(second);
+}
+
+function parsePrivateModeParams(params: string): Set<number> {
+  const modes = new Set<number>();
+  for (const param of params.split(";")) {
+    if (/^\d+$/.test(param)) {
+      modes.add(Number(param));
+    }
+  }
+  return modes;
+}
+
+export function terminalInputModeSupportsModifiedEnter(state: TerminalInputModeState): boolean {
+  return state.kittyKeyboardFlags > 0 || state.win32InputMode;
+}
+
+export function terminalInputModeStatesEqual(
+  left: TerminalInputModeState,
+  right: TerminalInputModeState,
+): boolean {
+  return (
+    left.kittyKeyboardFlags === right.kittyKeyboardFlags &&
+    left.win32InputMode === right.win32InputMode
+  );
+}
+
+export class TerminalInputModeTracker {
+  private kittyKeyboardFlags = 0;
+  private win32InputMode = false;
+  private readonly kittyKeyboardStack: number[] = [];
+  private pending = "";
+
+  feed(data: string): TerminalInputModeFeedResult {
+    if (data.length === 0) {
+      return { changed: false, responses: [] };
+    }
+
+    const text = `${this.pending}${data}`;
+    this.pending = "";
+
+    let changed = false;
+    const responses: string[] = [];
+    let consumedUntil = 0;
+
+    CSI_INPUT_MODE_SEQUENCE.lastIndex = 0;
+    for (;;) {
+      const match = CSI_INPUT_MODE_SEQUENCE.exec(text);
+      if (!match) {
+        break;
+      }
+      consumedUntil = CSI_INPUT_MODE_SEQUENCE.lastIndex;
+
+      if (match[4]) {
+        changed = this.applyPrivateModeSequence(match[3] ?? "", match[4]) || changed;
+        continue;
+      }
+
+      const result = this.applyKittyKeyboardSequence(match[1] ?? "", match[2] ?? "");
+      changed = changed || result.changed;
+      responses.push(...result.responses);
+    }
+
+    const tail = text.slice(consumedUntil);
+    const pendingStart = tail.lastIndexOf(`${ESC}[`);
+    if (pendingStart >= 0) {
+      const pending = tail.slice(pendingStart);
+      if (INCOMPLETE_CSI_INPUT_MODE_SEQUENCE.test(pending)) {
+        this.pending = pending;
+      }
+    }
+
+    return { changed, responses };
+  }
+
+  reset(): void {
+    this.kittyKeyboardFlags = 0;
+    this.win32InputMode = false;
+    this.kittyKeyboardStack.length = 0;
+    this.pending = "";
+  }
+
+  getState(): TerminalInputModeState {
+    return {
+      kittyKeyboardFlags: this.kittyKeyboardFlags,
+      win32InputMode: this.win32InputMode,
+    };
+  }
+
+  getKittyKeyboardFlags(): number {
+    return this.kittyKeyboardFlags;
+  }
+
+  supportsModifiedEnter(): boolean {
+    return terminalInputModeSupportsModifiedEnter(this.getState());
+  }
+
+  getPreamble(): string {
+    const parts: string[] = [];
+    if (this.kittyKeyboardFlags > 0) {
+      parts.push(`\x1b[=${this.kittyKeyboardFlags};1u`);
+    }
+    if (this.win32InputMode) {
+      parts.push("\x1b[?9001h");
+    }
+    return parts.join("");
+  }
+
+  private applyKittyKeyboardSequence(
+    prefix: string,
+    params: string,
+  ): { changed: boolean; responses: string[] } {
+    const previousFlags = this.kittyKeyboardFlags;
+
+    switch (prefix) {
+      case ">": {
+        this.kittyKeyboardStack.push(this.kittyKeyboardFlags);
+        this.kittyKeyboardFlags = parseFirstParam(params) ?? 1;
+        break;
+      }
+      case "=": {
+        const mode = parseSecondParam(params) ?? 1;
+        this.kittyKeyboardFlags = mode === 0 ? 0 : (parseFirstParam(params) ?? 0);
+        break;
+      }
+      case "<": {
+        const count = Math.max(1, parseFirstParam(params) ?? 1);
+        for (let index = 0; index < count; index += 1) {
+          this.kittyKeyboardFlags = this.kittyKeyboardStack.pop() ?? 0;
+        }
+        break;
+      }
+      case "?":
+        return {
+          changed: false,
+          responses: [`\x1b[?${this.kittyKeyboardFlags}u`],
+        };
+      default:
+        return { changed: false, responses: [] };
+    }
+
+    return {
+      changed: this.kittyKeyboardFlags !== previousFlags,
+      responses: [],
+    };
+  }
+
+  private applyPrivateModeSequence(params: string, final: string): boolean {
+    const modes = parsePrivateModeParams(params);
+    if (!modes.has(WIN32_INPUT_MODE)) {
+      return false;
+    }
+
+    const previous = this.win32InputMode;
+    this.win32InputMode = final === "h";
+    return this.win32InputMode !== previous;
+  }
+}

packages/server/src/shared/terminal-key-input.test.ts

@@ -19,15 +19,36 @@ describe("encodeTerminalKeyInput", () => {
     expect(encodeTerminalKeyInput({ key: "Backspace" })).toBe("\x7f");
   });
 
-  it("encodes shift+enter as kitty keyboard protocol CSI u", () => {
-    expect(encodeTerminalKeyInput({ key: "Enter", shift: true })).toBe("\x1b[13;2u");
+  it("keeps modified Enter as carriage return before enhanced input mode is active", () => {
+    expect(encodeTerminalKeyInput({ key: "Enter", shift: true })).toBe("\r");
   });
 
-  it("encodes enter with modifiers using CSI u", () => {
-    expect(encodeTerminalKeyInput({ key: "Enter", ctrl: true })).toBe("\x1b[13;5u");
-    expect(encodeTerminalKeyInput({ key: "Enter", alt: true })).toBe("\x1b[13;3u");
-    expect(encodeTerminalKeyInput({ key: "Enter", meta: true })).toBe("\x1b[13;9u");
-    expect(encodeTerminalKeyInput({ key: "Enter", shift: true, ctrl: true })).toBe("\x1b[13;6u");
+  it("encodes Enter with modifiers using CSI u after Kitty keyboard mode is active", () => {
+    const options = { inputMode: { kittyKeyboardFlags: 7, win32InputMode: false } };
+
+    expect(encodeTerminalKeyInput({ key: "Enter", shift: true }, options)).toBe("\x1b[13;2u");
+    expect(encodeTerminalKeyInput({ key: "Enter", ctrl: true }, options)).toBe("\x1b[13;5u");
+    expect(encodeTerminalKeyInput({ key: "Enter", alt: true }, options)).toBe("\x1b[13;3u");
+    expect(encodeTerminalKeyInput({ key: "Enter", meta: true }, options)).toBe("\x1b[13;9u");
+    expect(encodeTerminalKeyInput({ key: "Enter", shift: true, ctrl: true }, options)).toBe(
+      "\x1b[13;6u",
+    );
+  });
+
+  it("encodes Shift+Enter using Win32 input mode when ConPTY requests it", () => {
+    const options = { inputMode: { kittyKeyboardFlags: 0, win32InputMode: true } };
+
+    expect(encodeTerminalKeyInput({ key: "Enter", shift: true }, options)).toBe(
+      "\x1b[13;28;13;1;16;1_",
+    );
+  });
+
+  it("prefers Win32 input mode over CSI u when both modes are active", () => {
+    const options = { inputMode: { kittyKeyboardFlags: 7, win32InputMode: true } };
+
+    expect(encodeTerminalKeyInput({ key: "Enter", shift: true }, options)).toBe(
+      "\x1b[13;28;13;1;16;1_",
+    );
   });
 
   it("returns empty string for unsupported keys", () => {

packages/server/src/shared/terminal-key-input.ts

@@ -1,3 +1,5 @@
+import type { TerminalInputModeState } from "./terminal-input-mode.js";
+
 export interface TerminalKeyInput {
   key: string;
   ctrl?: boolean;
@@ -6,6 +8,14 @@ export interface TerminalKeyInput {
   meta?: boolean;
 }
 
+export interface TerminalKeyInputEncodingOptions {
+  inputMode?: TerminalInputModeState;
+}
+
+const WIN32_LEFT_ALT_PRESSED = 0x0002;
+const WIN32_LEFT_CTRL_PRESSED = 0x0008;
+const WIN32_SHIFT_PRESSED = 0x0010;
+
 function modifierParam(input: TerminalKeyInput): number {
   let value = 1;
   if (input.shift) value += 1;
@@ -15,6 +25,40 @@ function modifierParam(input: TerminalKeyInput): number {
   return value;
 }
 
+function win32ControlKeyState(input: TerminalKeyInput): number {
+  let value = 0;
+  if (input.shift) value += WIN32_SHIFT_PRESSED;
+  if (input.ctrl) value += WIN32_LEFT_CTRL_PRESSED;
+  if (input.alt) value += WIN32_LEFT_ALT_PRESSED;
+  return value;
+}
+
+function shouldUseWin32InputMode(
+  input: TerminalKeyInput,
+  options: TerminalKeyInputEncodingOptions,
+): boolean {
+  if (!options.inputMode?.win32InputMode) {
+    return false;
+  }
+  return Boolean(input.shift || input.ctrl || input.alt);
+}
+
+function shouldUseKittyKeyboardMode(
+  input: TerminalKeyInput,
+  options: TerminalKeyInputEncodingOptions,
+): boolean {
+  if ((options.inputMode?.kittyKeyboardFlags ?? 0) <= 0) {
+    return false;
+  }
+  return Boolean(input.shift || input.ctrl || input.alt || input.meta);
+}
+
+function encodeWin32EnterKeyInput(input: TerminalKeyInput): string {
+  const controlKeyState = win32ControlKeyState(input);
+  // ConPTY Win32 input mode: CSI Vk;Sc;Uc;Kd;Cs;Rc _
+  return `\x1b[13;28;13;1;${controlKeyState};1_`;
+}
+
 function applyAltLikePrefix(sequence: string, input: TerminalKeyInput): string {
   return input.alt ? `\x1b${sequence}` : sequence;
 }
@@ -144,7 +188,10 @@ function encodeNavigationKey(key: string, input: TerminalKeyInput): string | nul
   }
 }
 
-export function encodeTerminalKeyInput(input: TerminalKeyInput): string {
+export function encodeTerminalKeyInput(
+  input: TerminalKeyInput,
+  options: TerminalKeyInputEncodingOptions = {},
+): string {
   const key = input.key;
   if (!key) {
     return "";
@@ -157,7 +204,10 @@ export function encodeTerminalKeyInput(input: TerminalKeyInput): string {
   switch (key) {
     case "Enter": {
       const mod = modifierParam(input);
-      if (mod > 1) {
+      if (mod > 1 && shouldUseWin32InputMode(input, options)) {
+        return encodeWin32EnterKeyInput(input);
+      }
+      if (mod > 1 && shouldUseKittyKeyboardMode(input, options)) {
         return `\x1b[13;${mod}u`;
       }
       return "\r";