Harden SQLite migration and apply post-merge fixes

Migration hardening: - Back up JSON to $PASEO_HOME/backup/pre-migration/ before import - Deduplicate projects/workspaces by path before insert - Log per-batch progress during agent snapshot import - Clear error messages for corrupt JSON files - 5 new test cases for backup, dedup, progress, and error clarity Post-merge fixes: - Add worktreeRoot to session checkout result (type error fix) - Port reload agent tab action from main - Remove deleted useDelayedHistoryRefreshToast hook usage
2026-04-05 13:05:54 +07:00
parent fcfb9c99da
commit 2dd597f5f5
8 changed files with 402 additions and 43 deletions
@@ -1,8 +1,8 @@
 import os from "node:os";
 import path from "node:path";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";

-import { afterEach, beforeEach, describe, expect, test } from "vitest";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";

 import { createTestLogger } from "../../test-utils/test-logger.js";
 import { openPaseoDatabase, type PaseoDatabaseHandle } from "./sqlite-database.js";
@@ -192,6 +192,78 @@ describe("importLegacyAgentSnapshots", () => {
    expect(rows).toHaveLength(150);
    expect(rows.map((row) => row.agentId)).toContain("agent-149");
  });
+
+  test("creates backup of agent directory before import", async () => {
+    await seedWorkspace("/tmp/project");
+    writeLegacyAgentJson({
+      paseoHome,
+      relativePath: "agents/project-a/agent-1.json",
+      payload: createLegacyAgentJson(),
+    });
+
+    await importLegacyAgentSnapshots({
+      db: database.db,
+      paseoHome,
+      logger: createTestLogger(),
+    });
+
+    const backupPath = path.join(
+      paseoHome,
+      "backup",
+      "pre-migration",
+      "agents",
+      "project-a",
+      "agent-1.json",
+    );
+    expect(existsSync(backupPath)).toBe(true);
+    expect(JSON.parse(readFileSync(backupPath, "utf8"))).toMatchObject({
+      id: "agent-1",
+      cwd: "/tmp/project",
+    });
+  });
+
+  test("logs batch progress for large imports", async () => {
+    await seedWorkspace("/tmp/large-project");
+    const logger = createTestLogger();
+    const infoSpy = vi.spyOn(logger, "info");
+
+    for (let index = 0; index < 150; index += 1) {
+      writeLegacyAgentJson({
+        paseoHome,
+        relativePath: `agents/large-project/agent-${index}.json`,
+        payload: createLegacyAgentJson({
+          id: `agent-${index}`,
+          cwd: "/tmp/large-project",
+          runtimeInfo: {
+            provider: "codex",
+            sessionId: `session-${index}`,
+            model: "gpt-5.1-codex-mini",
+            modeId: "plan",
+          },
+        }),
+      });
+    }
+
+    await importLegacyAgentSnapshots({
+      db: database.db,
+      paseoHome,
+      logger,
+    });
+
+    const batchLogs = infoSpy.mock.calls.filter(
+      ([context, message]) => message === "Importing agent snapshot batch" && typeof context === "object",
+    );
+    expect(batchLogs.length).toBeGreaterThan(1);
+    expect(batchLogs[0]?.[0]).toMatchObject({
+      batch: 1,
+      totalBatches: batchLogs.length,
+    });
+    expect(batchLogs.at(-1)?.[0]).toMatchObject({
+      batch: batchLogs.length,
+      totalBatches: batchLogs.length,
+      rowsProcessed: 150,
+    });
+  });
 });

 function createLegacyAgentJson(overrides: Record<string, unknown> = {}): Record<string, unknown> {
@@ -40,7 +40,25 @@ export async function importLegacyAgentSnapshots(options: {
    };
  }

-  const records = await readLegacyAgentRecords(path.join(options.paseoHome, "agents"), options.logger);
+  const agentsDir = path.join(options.paseoHome, "agents");
+  if (!(await pathExists(agentsDir))) {
+    options.logger.info("Skipping legacy agent snapshot import because no legacy files exist");
+    return {
+      status: "skipped",
+      reason: "no-legacy-files",
+    };
+  }
+
+  await backupLegacyAgentDirectory({
+    sourceDir: agentsDir,
+    paseoHome: options.paseoHome,
+    logger: options.logger,
+  });
+
+  const { records, skippedCount } = await readLegacyAgentRecords(agentsDir, options.logger);
+  if (skippedCount > 0) {
+    options.logger.warn({ skippedCount }, "Skipped invalid agent JSON files during migration");
+  }
  if (records.length === 0) {
    options.logger.info("Skipping legacy agent snapshot import because no legacy files exist");
    return {
@@ -109,8 +127,15 @@ export async function importLegacyAgentSnapshots(options: {
      const workspaceId = workspaceIdsByDirectory.get(normalizeWorkspaceId(record.cwd));
      return workspaceId === undefined ? [] : [toAgentSnapshotRowValues({ record, workspaceId })];
    });
+    const totalBatches = Math.ceil(rows.length / MAX_AGENT_SNAPSHOT_ROWS_PER_INSERT);
    for (let startIndex = 0; startIndex < rows.length; startIndex += MAX_AGENT_SNAPSHOT_ROWS_PER_INSERT) {
      const batch = rows.slice(startIndex, startIndex + MAX_AGENT_SNAPSHOT_ROWS_PER_INSERT);
+      const batchNum = Math.floor(startIndex / MAX_AGENT_SNAPSHOT_ROWS_PER_INSERT) + 1;
+      const rowsProcessed = startIndex + batch.length;
+      options.logger.info(
+        { batch: batchNum, totalBatches, rowsProcessed },
+        "Importing agent snapshot batch",
+      );
      tx.insert(agentSnapshots).values(batch).run();
    }
  });
@@ -126,23 +151,29 @@ export async function importLegacyAgentSnapshots(options: {
  };
 }

-async function readLegacyAgentRecords(baseDir: string, logger: Logger): Promise<StoredAgentRecord[]> {
+async function readLegacyAgentRecords(baseDir: string, logger: Logger): Promise<{
+  records: StoredAgentRecord[];
+  skippedCount: number;
+}> {
  let entries: Array<import("node:fs").Dirent> = [];
  try {
    entries = await fs.readdir(baseDir, { withFileTypes: true });
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      return [];
+      return { records: [], skippedCount: 0 };
    }
    throw error;
  }

  const recordsById = new Map<string, StoredAgentRecord>();
+  let skippedCount = 0;
  for (const entry of entries) {
    if (entry.isFile() && entry.name.endsWith(".json")) {
      const record = await readRecordFile(path.join(baseDir, entry.name), logger);
      if (record) {
        recordsById.set(record.id, record);
+      } else {
+        skippedCount += 1;
      }
      continue;
    }
@@ -165,11 +196,16 @@ async function readLegacyAgentRecords(baseDir: string, logger: Logger): Promise<
      const record = await readRecordFile(path.join(baseDir, entry.name, childEntry.name), logger);
      if (record) {
        recordsById.set(record.id, record);
+      } else {
+        skippedCount += 1;
      }
    }
  }

-  return Array.from(recordsById.values());
+  return {
+    records: Array.from(recordsById.values()),
+    skippedCount,
+  };
 }

 async function readRecordFile(filePath: string, logger: Logger): Promise<StoredAgentRecord | null> {
@@ -186,3 +222,26 @@ async function hasAnyAgentSnapshotRows(db: PaseoDatabaseHandle["db"]): Promise<b
  const rows = await db.select({ count: count() }).from(agentSnapshots);
  return (rows[0]?.count ?? 0) > 0;
 }
+
+async function backupLegacyAgentDirectory(options: {
+  sourceDir: string;
+  paseoHome: string;
+  logger: Logger;
+}): Promise<void> {
+  const backupPath = path.join(options.paseoHome, "backup", "pre-migration", "agents");
+  await fs.mkdir(path.dirname(backupPath), { recursive: true });
+  await fs.cp(options.sourceDir, backupPath, { recursive: true });
+  options.logger.info({ backupPath }, "Backed up legacy agent snapshots before migration");
+}
+
+async function pathExists(targetPath: string): Promise<boolean> {
+  try {
+    await fs.access(targetPath);
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return false;
+    }
+    throw error;
+  }
+}
@@ -1,6 +1,6 @@
 import os from "node:os";
 import path from "node:path";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";

 import { afterEach, beforeEach, describe, expect, test } from "vitest";

@@ -173,6 +173,136 @@ describe("importLegacyProjectWorkspaceJson", () => {
    expect(await database.db.select().from(projects)).toEqual([]);
    expect(await database.db.select().from(workspaces)).toEqual([]);
  });
+
+  test("deduplicates projects with the same rootPath", async () => {
+    writeLegacyJson({
+      paseoHome,
+      projectsJson: [
+        {
+          projectId: "project-1",
+          rootPath: "/tmp/project-1",
+          kind: "git",
+          displayName: "First Project",
+          createdAt: "2026-03-01T00:00:00.000Z",
+          updatedAt: "2026-03-01T00:00:00.000Z",
+          archivedAt: null,
+        },
+        {
+          projectId: "project-2",
+          rootPath: "/tmp/project-1",
+          kind: "git",
+          displayName: "Replacement Project",
+          createdAt: "2026-03-01T00:00:00.000Z",
+          updatedAt: "2026-03-03T00:00:00.000Z",
+          archivedAt: null,
+        },
+      ],
+      workspacesJson: [
+        {
+          workspaceId: "workspace-1",
+          projectId: "project-2",
+          cwd: "/tmp/project-1",
+          kind: "local_checkout",
+          displayName: "main",
+          createdAt: "2026-03-01T00:00:00.000Z",
+          updatedAt: "2026-03-03T00:00:00.000Z",
+          archivedAt: null,
+        },
+      ],
+    });
+
+    const result = await importLegacyProjectWorkspaceJson({
+      db: database.db,
+      paseoHome,
+      logger: createTestLogger(),
+    });
+
+    expect(result).toEqual({
+      status: "imported",
+      importedProjects: 1,
+      importedWorkspaces: 1,
+    });
+    const projectRows = await database.db.select().from(projects);
+    expect(projectRows).toHaveLength(1);
+    expect(projectRows[0]).toEqual(
+      expect.objectContaining({
+        directory: "/tmp/project-1",
+        displayName: "Replacement Project",
+      }),
+    );
+  });
+
+  test("creates backup of JSON files before import", async () => {
+    writeLegacyJson({
+      paseoHome,
+      projectsJson: [
+        {
+          projectId: "project-1",
+          rootPath: "/tmp/project-1",
+          kind: "git",
+          displayName: "Project One",
+          createdAt: "2026-03-01T00:00:00.000Z",
+          updatedAt: "2026-03-02T00:00:00.000Z",
+          archivedAt: null,
+        },
+      ],
+      workspacesJson: [
+        {
+          workspaceId: "workspace-1",
+          projectId: "project-1",
+          cwd: "/tmp/project-1",
+          kind: "local_checkout",
+          displayName: "main",
+          createdAt: "2026-03-01T00:00:00.000Z",
+          updatedAt: "2026-03-02T00:00:00.000Z",
+          archivedAt: null,
+        },
+      ],
+    });
+
+    await importLegacyProjectWorkspaceJson({
+      db: database.db,
+      paseoHome,
+      logger: createTestLogger(),
+    });
+
+    const backupDir = path.join(paseoHome, "backup", "pre-migration");
+    const projectsBackupPath = path.join(backupDir, "projects.json");
+    const workspacesBackupPath = path.join(backupDir, "workspaces.json");
+    expect(existsSync(projectsBackupPath)).toBe(true);
+    expect(existsSync(workspacesBackupPath)).toBe(true);
+    expect(JSON.parse(readFileSync(projectsBackupPath, "utf8"))).toHaveLength(1);
+    expect(JSON.parse(readFileSync(workspacesBackupPath, "utf8"))).toHaveLength(1);
+  });
+
+  test("produces clear error message for corrupt project JSON", async () => {
+    writeLegacyJson({
+      paseoHome,
+      projectsJson: [
+        {
+          projectId: "project-1",
+          rootPath: 123,
+          kind: "git",
+          displayName: "Project One",
+          createdAt: "2026-03-01T00:00:00.000Z",
+          updatedAt: "2026-03-02T00:00:00.000Z",
+          archivedAt: null,
+        },
+      ],
+      workspacesJson: [],
+    });
+
+    await expect(
+      importLegacyProjectWorkspaceJson({
+        db: database.db,
+        paseoHome,
+        logger: createTestLogger(),
+      }),
+    ).rejects.toThrow(
+      `Failed to parse ${path.join(paseoHome, "projects", "projects.json")}. ` +
+        "The file may be corrupted.",
+    );
+  });
 });

 function writeLegacyJson(input: {
@@ -49,11 +49,7 @@ export async function importLegacyProjectWorkspaceJson(options: {
 }): Promise<LegacyProjectWorkspaceImportResult> {
  const projectsPath = path.join(options.paseoHome, "projects", "projects.json");
  const workspacesPath = path.join(options.paseoHome, "projects", "workspaces.json");
-  const [projectRows, workspaceRows, databaseHasRows] = await Promise.all([
-    readLegacyProjects(projectsPath),
-    readLegacyWorkspaces(workspacesPath),
-    hasAnyProjectWorkspaceRows(options.db),
-  ]);
+  const databaseHasRows = await hasAnyProjectWorkspaceRows(options.db);

  if (databaseHasRows) {
    options.logger.info("Skipping legacy project/workspace JSON import because the DB is not empty");
@@ -63,6 +59,30 @@ export async function importLegacyProjectWorkspaceJson(options: {
    };
  }

+  const [projectsExists, workspacesExists] = await Promise.all([
+    pathExists(projectsPath),
+    pathExists(workspacesPath),
+  ]);
+  if (!projectsExists && !workspacesExists) {
+    options.logger.info("Skipping legacy project/workspace JSON import because no legacy files exist");
+    return {
+      status: "skipped",
+      reason: "no-legacy-files",
+    };
+  }
+
+  await backupLegacyProjectWorkspaceJson({
+    projectsPath,
+    workspacesPath,
+    paseoHome: options.paseoHome,
+    logger: options.logger,
+  });
+
+  const [projectRows, workspaceRows] = await Promise.all([
+    readLegacyProjects(projectsPath),
+    readLegacyWorkspaces(workspacesPath),
+  ]);
+
  if (projectRows.length === 0 && workspaceRows.length === 0) {
    options.logger.info("Skipping legacy project/workspace JSON import because no legacy files exist");
    return {
@@ -71,10 +91,18 @@ export async function importLegacyProjectWorkspaceJson(options: {
    };
  }

+  const dedupedProjects = [...new Map(projectRows.map((project) => [project.rootPath, project])).values()];
+  const dedupedWorkspaces = [...new Map(workspaceRows.map((workspace) => [workspace.cwd, workspace])).values()];
+
+  options.logger.info(
+    { projects: dedupedProjects.length, workspaces: dedupedWorkspaces.length },
+    "Starting legacy project/workspace import",
+  );
+
  options.db.transaction((tx) => {
    // Insert projects, mapping old format to new schema
    const projectDirectoryToId = new Map<string, number>();
-    for (const legacy of projectRows) {
+    for (const legacy of dedupedProjects) {
      const row = tx
        .insert(projects)
        .values({
@@ -100,7 +128,7 @@ export async function importLegacyProjectWorkspaceJson(options: {
    }

    // Insert workspaces, resolving project FK
-    for (const legacy of workspaceRows) {
+    for (const legacy of dedupedWorkspaces) {
      const projectId = legacyProjectIdToNewId.get(legacy.projectId);
      if (projectId === undefined) {
        throw new Error(`Legacy workspace ${legacy.workspaceId} references unknown project ${legacy.projectId}`);
@@ -125,27 +153,49 @@ export async function importLegacyProjectWorkspaceJson(options: {

  options.logger.info(
    {
-      importedProjects: projectRows.length,
-      importedWorkspaces: workspaceRows.length,
+      importedProjects: dedupedProjects.length,
+      importedWorkspaces: dedupedWorkspaces.length,
    },
    "Imported legacy project/workspace JSON into the database",
  );

  return {
    status: "imported",
-    importedProjects: projectRows.length,
-    importedWorkspaces: workspaceRows.length,
+    importedProjects: dedupedProjects.length,
+    importedWorkspaces: dedupedWorkspaces.length,
  };
 }

 async function readLegacyProjects(filePath: string) {
  const raw = await readOptionalJsonFile(filePath);
-  return raw ? z.array(LegacyProjectSchema).parse(raw) : [];
+  if (!raw) {
+    return [];
+  }
+  try {
+    return z.array(LegacyProjectSchema).parse(raw);
+  } catch (error) {
+    throw new Error(
+      `Failed to parse ${filePath}. The file may be corrupted. ` +
+        `Check the file and fix or remove invalid entries. ` +
+        `Original error: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
 }

 async function readLegacyWorkspaces(filePath: string) {
  const raw = await readOptionalJsonFile(filePath);
-  return raw ? z.array(LegacyWorkspaceSchema).parse(raw) : [];
+  if (!raw) {
+    return [];
+  }
+  try {
+    return z.array(LegacyWorkspaceSchema).parse(raw);
+  } catch (error) {
+    throw new Error(
+      `Failed to parse ${filePath}. The file may be corrupted. ` +
+        `Check the file and fix or remove invalid entries. ` +
+        `Original error: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
 }

 async function readOptionalJsonFile(filePath: string): Promise<unknown | null> {
@@ -170,3 +220,34 @@ async function hasAnyProjectWorkspaceRows(db: PaseoDatabaseHandle["db"]): Promis
  const workspaceCount = workspaceCountRows[0]?.count ?? 0;
  return projectCount > 0 || workspaceCount > 0;
 }
+
+async function backupLegacyProjectWorkspaceJson(options: {
+  projectsPath: string;
+  workspacesPath: string;
+  paseoHome: string;
+  logger: Logger;
+}): Promise<void> {
+  const backupDir = path.join(options.paseoHome, "backup", "pre-migration");
+  await fs.mkdir(backupDir, { recursive: true });
+
+  if (await pathExists(options.projectsPath)) {
+    await fs.copyFile(options.projectsPath, path.join(backupDir, "projects.json"));
+  }
+  if (await pathExists(options.workspacesPath)) {
+    await fs.copyFile(options.workspacesPath, path.join(backupDir, "workspaces.json"));
+  }
+
+  options.logger.info({ backupPath: backupDir }, "Backed up legacy project/workspace JSON before migration");
+}
+
+async function pathExists(targetPath: string): Promise<boolean> {
+  try {
+    await fs.access(targetPath);
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return false;
+    }
+    throw error;
+  }
+}
@@ -1290,6 +1290,7 @@ export class Session {
            isGit: false as const,
            currentBranch: null,
            remoteUrl: null,
+            worktreeRoot: null,
            isPaseoOwnedWorktree: false as const,
            mainRepoRoot: null,
          }
@@ -1299,6 +1300,7 @@ export class Session {
              isGit: true as const,
              currentBranch: workspace.displayName,
              remoteUrl: project.gitRemote,
+              worktreeRoot: workspace.directory,
              isPaseoOwnedWorktree: true as const,
              mainRepoRoot: project.directory,
            }
@@ -1307,6 +1309,7 @@ export class Session {
              isGit: true as const,
              currentBranch: workspace.displayName,
              remoteUrl: project.gitRemote,
+              worktreeRoot: workspace.directory,
              isPaseoOwnedWorktree: false as const,
              mainRepoRoot: null,
            };