HEX
Server: Apache/2.4.46 (Win64) OpenSSL/1.1.1j PHP/8.4.25
System: Windows NT DESKTOP-4TAV2RJ 10.0 build 19045 (Windows 10) AMD64
User: fred (0)
PHP: 8.4.25
Disabled: NONE
Upload Files
File: C:/Users/fred/.codex/.tmp/plugins/plugins/plugin-eval/src/evaluators/skill.js
import path from "node:path";

import { createArtifact, createCheck, createMetric } from "../core/schema.js";
import { parseFrontmatter } from "../lib/frontmatter.js";
import { pathExists, readText, relativePath, walkFiles } from "../lib/files.js";

const ALLOWED_FRONTMATTER_KEYS = new Set([
  "name",
  "description",
  "license",
  "allowed-tools",
  "metadata",
  "disable-model-invocation",
  "user-invocable",
  "context",
  "agent",
]);

function countCodeFences(markdown) {
  return (markdown.match(/```/g) || []).length / 2;
}

function findRelativeLinks(markdown) {
  return [...markdown.matchAll(/\[[^\]]+\]\(([^)]+)\)/g)]
    .map((match) => match[1])
    .filter(
      (target) =>
        !target.startsWith("http://") &&
        !target.startsWith("https://") &&
        !target.startsWith("app://") &&
        !target.startsWith("plugin://") &&
        !target.startsWith("rules://") &&
        !target.startsWith("mailto:") &&
        !target.startsWith("#"),
    );
}

function isHyphenCase(value) {
  return /^[a-z0-9-]+$/.test(value) && !value.startsWith("-") && !value.endsWith("-") && !value.includes("--");
}

export async function evaluateSkill(skillRoot, options = {}) {
  const prefix = options.prefix ? `${options.prefix}:` : "";
  const skillPath = path.join(skillRoot, "SKILL.md");
  const targetPath = relativePath(process.cwd(), skillRoot);
  const checks = [];
  const metrics = [];
  const artifacts = [];

  if (!(await pathExists(skillPath))) {
    checks.push(
      createCheck({
        id: `${prefix}skill-file-missing`,
        category: "skill-structure",
        severity: "error",
        status: "fail",
        message: "The target skill directory is missing SKILL.md.",
        evidence: [targetPath],
        remediation: ["Add SKILL.md to the skill root."],
        targetPath,
      }),
    );
    return { checks, metrics, artifacts };
  }

  const content = await readText(skillPath);
  const lines = content.replace(/\r\n/g, "\n").split("\n");
  const parsed = parseFrontmatter(content);
  const relativeLinks = findRelativeLinks(content);
  const supportFiles = (await walkFiles(skillRoot)).filter((filePath) => filePath !== skillPath);

  if (parsed.errors.length > 0) {
    checks.push(
      createCheck({
        id: `${prefix}frontmatter-invalid`,
        category: "skill-structure",
        severity: "error",
        status: "fail",
        message: "The skill frontmatter could not be parsed.",
        evidence: parsed.errors,
        remediation: ["Fix the YAML frontmatter at the top of SKILL.md."],
        targetPath,
      }),
    );
  }

  const frontmatter = parsed.data || {};
  const unexpectedKeys = Object.keys(frontmatter).filter((key) => !ALLOWED_FRONTMATTER_KEYS.has(key));
  if (unexpectedKeys.length > 0) {
    checks.push(
      createCheck({
        id: `${prefix}frontmatter-extra-keys`,
        category: "best-practice",
        severity: "warning",
        status: "warn",
        message: "The skill frontmatter contains keys outside the common Codex skill conventions.",
        evidence: unexpectedKeys.map((key) => `Unexpected key: ${key}`),
        remediation: ["Remove non-standard frontmatter keys or move the metadata into references."],
        targetPath,
      }),
    );
  }

  if (!frontmatter.name) {
    checks.push(
      createCheck({
        id: `${prefix}name-missing`,
        category: "skill-structure",
        severity: "error",
        status: "fail",
        message: "The skill frontmatter is missing `name`.",
        evidence: [targetPath],
        remediation: ["Add a hyphen-case `name` field to the frontmatter."],
        targetPath,
      }),
    );
  } else if (!isHyphenCase(frontmatter.name)) {
    checks.push(
      createCheck({
        id: `${prefix}name-not-hyphen-case`,
        category: "skill-structure",
        severity: "error",
        status: "fail",
        message: "The skill name should be lowercase hyphen-case.",
        evidence: [`Current name: ${frontmatter.name}`],
        remediation: ["Rename the skill using lowercase letters, digits, and single hyphens only."],
        targetPath,
      }),
    );
  }

  if (!frontmatter.description) {
    checks.push(
      createCheck({
        id: `${prefix}description-missing`,
        category: "skill-structure",
        severity: "error",
        status: "fail",
        message: "The skill frontmatter is missing `description`.",
        evidence: [targetPath],
        remediation: ["Add a description that explains what the skill does and when to use it."],
        targetPath,
      }),
    );
  } else {
    if (frontmatter.description.length > 1024) {
      checks.push(
        createCheck({
          id: `${prefix}description-too-long`,
          category: "budget",
          severity: "warning",
          status: "warn",
          message: "The skill description is long enough to create unnecessary always-loaded context cost.",
          evidence: [`Description length: ${frontmatter.description.length} characters`],
          remediation: ["Shorten the description and push details into SKILL.md or references."],
          targetPath,
        }),
      );
    }
    if (!/use when/i.test(frontmatter.description)) {
      checks.push(
        createCheck({
          id: `${prefix}description-trigger-weak`,
          category: "best-practice",
          severity: "warning",
          status: "warn",
          message: "The description does not clearly advertise when the skill should trigger.",
          evidence: ["Descriptions are the primary auto-load surface in Codex."],
          remediation: ["Rewrite the description to include a clear 'Use when ...' trigger sentence."],
          targetPath,
        }),
      );
    }
  }

  if (lines.length > 800) {
    checks.push(
      createCheck({
        id: `${prefix}skill-too-large`,
        category: "budget",
        severity: "error",
        status: "fail",
        message: "SKILL.md is extremely large for an always-loaded invocation surface.",
        evidence: [`Line count: ${lines.length}`],
        remediation: ["Move large details into references/ and keep SKILL.md focused on the core workflow."],
        targetPath,
      }),
    );
  } else if (lines.length > 500) {
    checks.push(
      createCheck({
        id: `${prefix}skill-large`,
        category: "budget",
        severity: "warning",
        status: "warn",
        message: "SKILL.md exceeds the recommended compact size for progressive disclosure.",
        evidence: [`Line count: ${lines.length}`],
        remediation: ["Trim repetitive detail and move long variants into references/."],
        targetPath,
      }),
    );
  }

  if (lines.length > 350 && supportFiles.filter((filePath) => filePath.includes("/references/")).length === 0) {
    checks.push(
      createCheck({
        id: `${prefix}progressive-disclosure-missing`,
        category: "best-practice",
        severity: "warning",
        status: "warn",
        message: "The skill is getting large without using references for progressive disclosure.",
        evidence: ["Large skills are easier to maintain when variants live under references/."],
        remediation: ["Move deep detail or edge-case variants into references/ and link them from SKILL.md."],
        targetPath,
      }),
    );
  }

  const brokenLinks = [];
  for (const target of relativeLinks) {
    const candidatePath = path.resolve(skillRoot, target);
    if (!(await pathExists(candidatePath))) {
      brokenLinks.push(target);
    }
  }
  if (brokenLinks.length > 0) {
    checks.push(
      createCheck({
        id: `${prefix}broken-relative-links`,
        category: "skill-structure",
        severity: "error",
        status: "fail",
        message: "The skill contains relative links that do not resolve inside the skill directory.",
        evidence: brokenLinks,
        remediation: ["Fix or remove broken links in SKILL.md."],
        targetPath,
      }),
    );
  }

  const extraDocs = supportFiles
    .map((filePath) => path.basename(filePath))
    .filter((name) => ["README.md", "CHANGELOG.md", "CONTRIBUTING.md"].includes(name));
  if (extraDocs.length > 0) {
    checks.push(
      createCheck({
        id: `${prefix}extra-doc-files`,
        category: "best-practice",
        severity: "warning",
        status: "warn",
        message: "The skill includes extra documentation files that usually belong in references/ or outside the skill bundle.",
        evidence: extraDocs,
        remediation: ["Move reusable guidance into references/ and avoid extra top-level docs inside the skill bundle."],
        targetPath,
      }),
    );
  }

  metrics.push(
    createMetric({
      id: `${prefix}skill_line_count`,
      category: "budget",
      value: lines.length,
      unit: "lines",
      band: lines.length > 500 ? "heavy" : lines.length > 350 ? "moderate" : "good",
      targetPath,
    }),
    createMetric({
      id: `${prefix}description_length_chars`,
      category: "budget",
      value: String(frontmatter.description || "").length,
      unit: "chars",
      band:
        String(frontmatter.description || "").length > 1024
          ? "heavy"
          : String(frontmatter.description || "").length > 512
            ? "moderate"
            : "good",
      targetPath,
    }),
    createMetric({
      id: `${prefix}relative_link_count`,
      category: "documentation",
      value: relativeLinks.length,
      unit: "links",
      band: relativeLinks.length > 10 ? "moderate" : "good",
      targetPath,
    }),
    createMetric({
      id: `${prefix}code_fence_count`,
      category: "readability",
      value: countCodeFences(content),
      unit: "blocks",
      band: countCodeFences(content) > 8 ? "moderate" : "good",
      targetPath,
    }),
    createMetric({
      id: `${prefix}support_file_count`,
      category: "documentation",
      value: supportFiles.length,
      unit: "files",
      band: supportFiles.length > 0 ? "good" : "info",
      targetPath,
    }),
  );

  artifacts.push(
    createArtifact({
      id: `${prefix}skill-link-inventory`,
      type: "inventory",
      label: "Skill relative links",
      description: "Relative links found in SKILL.md.",
      data: {
        links: relativeLinks,
      },
    }),
  );

  return { checks, metrics, artifacts };
}