Variable manifestConst

Type Declaration

• Readonlyid: "benchmarks"

• Readonlyjobs: readonly [{ gpu: "exclusive"; isolation: "task"; kind: "eval-serve" }]

• Readonlypanel: {
      entry: "@mlx-bun/module-benchmarks/panel";
      path: "/benchmarks";
      tag: "mlx-benchmarks-panel";
      title: "Benchmarks";
  }

• Readonlyrequires: readonly ["jobs", "storage", "modelHost", "catalog"]

• Readonlyroutes: readonly [
      {
          id: "tasks";
          method: "GET";
          path: "/tasks";
          response: "json";
          summary: "The tasks a run can evaluate: their sets, pinned datasets and whether each dataset file is present";
      },
      {
          id: "run-start";
          method: "POST";
          path: "/runs";
          response: "json";
          summary: "Start an evaluation run from the JSON body (`tasks`: ids or a set name, default `smoketest`; `model`: a catalog id or query, default the served model; `enableThinking`); it takes the GPU exclusively until it ends";
      },
      {
          id: "history";
          method: "GET";
          path: "/runs";
          response: "json";
          summary: "Finished evaluation runs, newest first, as compact per-task scores (`limit` query)";
      },
      {
          id: "run";
          method: "GET";
          path: "/runs/:id";
          response: "json";
          summary: "One finished evaluation run";
      },
      {
          id: "compare";
          method: "GET";
          path: "/compare";
          response: "json";
          summary: "The runner's paired comparison of two finished runs (`base` and `candidate` ids)";
      },
      {
          id: "jobs";
          method: "GET";
          path: "/jobs";
          response: "json";
          summary: "Recent evaluation jobs";
      },
      {
          id: "job";
          method: "GET";
          path: "/jobs/:id";
          response: "json";
          summary: "One evaluation job";
      },
      {
          id: "job-cancel";
          method: "DELETE";
          path: "/jobs/:id";
          response: "json";
          summary: "Stop a running evaluation job";
      },
  ]

• Readonlystorage: readonly [
      {
          key: "history";
          kind: "directory";
          path: "benchmarks/history";
          purpose: "Compact results of finished evaluation runs, one JSON file each";
      },
      {
          key: "runs";
          kind: "directory";
          path: "benchmarks/runs";
          purpose: "Each run's own directory: result.json with provenance, samples.jsonl, report.md, the server's log";
      },
      {
          key: "plans";
          kind: "directory";
          path: "benchmarks/plans";
          purpose: "The pinned plan each run was made from";
      },
      {
          key: "data";
          kind: "directory";
          path: "benchmarks/data";
          purpose: "The sha256-pinned evaluation datasets, placed here by the user; never downloaded";
      },
  ]

• Readonlysummary: "Answer-quality benchmarks over the capability evaluation runner (GSM8K, MMLU, IFEval, BFCL, HumanEval, HashHop): runs a plan against the served or a named model as a job, keeps the compact results history, and compares two runs."

• Readonlytitle: "Benchmarks"