HEX
Server: Apache/2.4.46 (Win64) OpenSSL/1.1.1j PHP/8.4.25
System: Windows NT DESKTOP-4TAV2RJ 10.0 build 19045 (Windows 10) AMD64
User: fred (0)
PHP: 8.4.25
Disabled: NONE
Upload Files
File: C:/Users/fred/.codex/.tmp/plugins/plugins/plugin-eval/src/core/benchmark-workspace.js
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";

import { isProbablyTextFile, isDirectory, readText, relativePath } from "../lib/files.js";

const execFileAsync = promisify(execFile);

const SNAPSHOT_IGNORED_DIRS = new Set([
  ".git",
  "node_modules",
  "dist",
  "build",
  ".next",
  ".turbo",
  ".cache",
  ".tmp",
  ".venv",
  "venv",
  "__pycache__",
  ".pytest_cache",
  ".plugin-eval",
]);

function testFilePattern(filePath) {
  return /(^|\/)(tests?|__tests__)\/|(\.test|\.spec)\.|(^|\/)test_[^/]+\.py$/.test(filePath);
}

async function copyDirectory(sourcePath, destinationPath) {
  await fs.mkdir(path.dirname(destinationPath), { recursive: true });
  await fs.cp(sourcePath, destinationPath, { recursive: true });
}

async function copyIfExists(sourcePath, destinationPath) {
  try {
    await fs.access(sourcePath);
  } catch {
    return false;
  }

  await fs.mkdir(path.dirname(destinationPath), { recursive: true });
  await fs.copyFile(sourcePath, destinationPath);
  return true;
}

async function resolveGitRoot(sourcePath) {
  const { stdout } = await execFileAsync("git", ["-C", sourcePath, "rev-parse", "--show-toplevel"]);
  return stdout.trim();
}

async function createWorkspaceCopy(sourcePath, workspacePath) {
  await copyDirectory(sourcePath, workspacePath);
  return {
    workspacePath,
    cleanup: async () => {
      await fs.rm(workspacePath, { recursive: true, force: true });
    },
  };
}

async function createWorkspaceFromGitWorktree(sourcePath, workspacePath) {
  const repoRoot = await resolveGitRoot(sourcePath);
  await execFileAsync("git", ["-C", repoRoot, "worktree", "add", "--detach", workspacePath]);
  return {
    workspacePath,
    cleanup: async () => {
      await execFileAsync("git", ["-C", repoRoot, "worktree", "remove", "--force", workspacePath]);
    },
  };
}

async function provisionSkillInstall(target, codexHomePath) {
  const skillPath = path.join(codexHomePath, "skills", target.name);
  await copyDirectory(target.path, skillPath);
  return skillPath;
}

async function provisionPluginInstall(target, workspacePath) {
  const pluginsRoot = path.join(workspacePath, "plugins");
  const installPath = path.join(pluginsRoot, target.name);
  await copyDirectory(target.path, installPath);

  const marketplacePath = path.join(workspacePath, ".agents", "plugins", "marketplace.json");
  const marketplace = {
    name: "plugin-eval-benchmark",
    interface: {
      displayName: "Plugin Eval Benchmark",
    },
    plugins: [
      {
        name: target.name,
        source: {
          source: "local",
          path: `./plugins/${target.name}`,
        },
        policy: {
          installation: "AVAILABLE",
          authentication: "ON_INSTALL",
        },
        category: "Developer Tools",
      },
    ],
  };

  await fs.mkdir(path.dirname(marketplacePath), { recursive: true });
  await fs.writeFile(marketplacePath, `${JSON.stringify(marketplace, null, 2)}\n`, "utf8");
  return installPath;
}

async function seedCodexHome(codexHomePath) {
  const sourceCodexHome = process.env.PLUGIN_EVAL_CODEX_HOME_SOURCE
    ? path.resolve(process.env.PLUGIN_EVAL_CODEX_HOME_SOURCE)
    : path.join(os.homedir(), ".codex");

  await fs.mkdir(codexHomePath, { recursive: true });
  await Promise.all([
    copyIfExists(path.join(sourceCodexHome, "auth.json"), path.join(codexHomePath, "auth.json")),
    copyIfExists(path.join(sourceCodexHome, "config.toml"), path.join(codexHomePath, "config.toml")),
  ]);
}

export function defaultTargetProvisioningMode(target) {
  if (target.kind === "skill") {
    return "isolated-skill-home";
  }
  if (target.kind === "plugin") {
    return "workspace-plugin-marketplace";
  }
  throw new Error("Benchmarking only supports Codex skills and plugins.");
}

export async function provisionBenchmarkWorkspace({ target, config, scenarioId }) {
  const sourcePath = path.resolve(config.workspace.sourcePath);
  if (!(await isDirectory(sourcePath))) {
    throw new Error(`Benchmark workspace.sourcePath must be a directory: ${sourcePath}`);
  }

  const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), `plugin-eval-${scenarioId}-`));
  const workspacePath = path.join(tempRoot, "workspace");
  const homePath = path.join(tempRoot, "home");
  const codexHomePath = path.join(homePath, ".codex");
  const setupMode = config.workspace.setupMode || "copy";

  await seedCodexHome(codexHomePath);

  const workspace =
    setupMode === "git-worktree"
      ? await createWorkspaceFromGitWorktree(sourcePath, workspacePath)
      : await createWorkspaceCopy(sourcePath, workspacePath);

  let installedTargetPath = null;
  if (config.targetProvisioning.mode === "isolated-skill-home") {
    installedTargetPath = await provisionSkillInstall(target, codexHomePath);
  } else if (config.targetProvisioning.mode === "workspace-plugin-marketplace") {
    installedTargetPath = await provisionPluginInstall(target, workspacePath);
  } else {
    throw new Error(`Unsupported target provisioning mode: ${config.targetProvisioning.mode}`);
  }

  return {
    tempRoot,
    workspacePath,
    homePath,
    codexHomePath,
    installedTargetPath,
    setupMode,
    cleanup: async () => {
      await workspace.cleanup();
      await fs.rm(tempRoot, { recursive: true, force: true });
    },
  };
}

async function hashFile(filePath) {
  const hash = createHash("sha1");
  hash.update(await fs.readFile(filePath));
  return hash.digest("hex");
}

async function visitSnapshot(rootPath, currentPath, entries) {
  const directoryEntries = await fs.readdir(currentPath, { withFileTypes: true });

  for (const entry of directoryEntries) {
    const entryPath = path.join(currentPath, entry.name);
    const relativeEntryPath = relativePath(rootPath, entryPath);

    if (entry.isDirectory()) {
      if (SNAPSHOT_IGNORED_DIRS.has(entry.name)) {
        continue;
      }
      await visitSnapshot(rootPath, entryPath, entries);
      continue;
    }

    if (!entry.isFile()) {
      continue;
    }

    const stats = await fs.stat(entryPath);
    const isText = isProbablyTextFile(entryPath);
    const text = isText ? await readText(entryPath) : null;

    entries.set(relativeEntryPath, {
      path: relativeEntryPath,
      absolutePath: entryPath,
      size: stats.size,
      hash: await hashFile(entryPath),
      isText,
      lineCount: text === null ? null : text.replace(/\r\n/g, "\n").split("\n").length,
    });
  }
}

export async function snapshotWorkspace(rootPath) {
  const entries = new Map();
  await visitSnapshot(rootPath, rootPath, entries);
  return entries;
}

export function diffWorkspaceSnapshots(beforeSnapshot, afterSnapshot) {
  const added = [];
  const modified = [];
  const deleted = [];

  for (const [filePath, afterEntry] of afterSnapshot.entries()) {
    const beforeEntry = beforeSnapshot.get(filePath);
    if (!beforeEntry) {
      added.push({
        path: filePath,
        status: "added",
        isText: afterEntry.isText,
        lineCountBefore: 0,
        lineCountAfter: afterEntry.lineCount,
        lineDelta: typeof afterEntry.lineCount === "number" ? afterEntry.lineCount : null,
      });
      continue;
    }

    if (beforeEntry.hash !== afterEntry.hash) {
      modified.push({
        path: filePath,
        status: "modified",
        isText: afterEntry.isText || beforeEntry.isText,
        lineCountBefore: beforeEntry.lineCount,
        lineCountAfter: afterEntry.lineCount,
        lineDelta:
          typeof beforeEntry.lineCount === "number" && typeof afterEntry.lineCount === "number"
            ? afterEntry.lineCount - beforeEntry.lineCount
            : null,
      });
    }
  }

  for (const [filePath, beforeEntry] of beforeSnapshot.entries()) {
    if (!afterSnapshot.has(filePath)) {
      deleted.push({
        path: filePath,
        status: "deleted",
        isText: beforeEntry.isText,
        lineCountBefore: beforeEntry.lineCount,
        lineCountAfter: 0,
        lineDelta: typeof beforeEntry.lineCount === "number" ? -beforeEntry.lineCount : null,
      });
    }
  }

  return {
    added: added.sort((a, b) => a.path.localeCompare(b.path)),
    modified: modified.sort((a, b) => a.path.localeCompare(b.path)),
    deleted: deleted.sort((a, b) => a.path.localeCompare(b.path)),
  };
}

export function summarizeWorkspaceDiff(diff) {
  const changedFiles = [...diff.added, ...diff.modified];
  return {
    addedFileCount: diff.added.length,
    modifiedFileCount: diff.modified.length,
    deletedFileCount: diff.deleted.length,
    changedFileCount: changedFiles.length,
    generatedFileCount: diff.added.length,
    generatedTestFileCount: diff.added.filter((entry) => testFilePattern(entry.path)).length,
    changedTestFileCount: changedFiles.filter((entry) => testFilePattern(entry.path)).length,
    changedTextLineDelta: changedFiles.reduce((sum, entry) => sum + (entry.lineDelta || 0), 0),
    changedFiles,
    allChanges: [...diff.added, ...diff.modified, ...diff.deleted],
  };
}