Basic reporter to output scores with each eval (#12)

Author: dcramer

package.json

@@ -5,14 +5,17 @@
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
-  "files": [
-    "dist"
-  ],
+  "files": ["dist"],
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
       "require": "./dist/index.js",
       "import": "./dist/index.mjs"
+    },
+    "./reporter": {
+      "types": "./dist/reporter.d.ts",
+      "require": "./dist/reporter.js",
+      "import": "./dist/reporter.mjs"
     }
   },
   "scripts": {
@@ -37,6 +40,7 @@
   "homepage": "https://github.com/getsentry/vitest-evals#readme",
   "description": "",
   "peerDependencies": {
+    "tinyrainbow": "*",
     "vitest": "*"
   },
   "devDependencies": {
@@ -46,6 +50,7 @@
     "lint-staged": "^15.5.0",
     "simple-git-hooks": "^2.12.1",
     "tsup": "^8.4.0",
+    "tinyrainbow": "*",
     "typescript": "^5.8.3",
     "vitest": "*"
   },

src/reporter.ts

@@ -0,0 +1,26 @@
+// import type { RunnerTask, RunnerTestFile } from "vitest";
+import { DefaultReporter } from "vitest/reporters";
+import c from "tinyrainbow";
+
+export default class DefaultEvalReporter extends DefaultReporter {
+  protected override printTestCase(moduleState: any, test: any): void {
+    const meta = test.meta();
+    const testResult = test.result();
+
+    if (!meta.eval || testResult.state === "failed") {
+      super.printTestCase(moduleState, test);
+      return;
+    }
+
+    const padding = this.getTestIndentation(test.task);
+    const colorFn =
+      meta.eval.avgScore < 0.5
+        ? c.red
+        : meta.eval.avgScore < 0.75
+          ? c.yellow
+          : c.green;
+    this.log(
+      `${padding}  ${this.getTestName(test.task, c.dim(" > "))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`,
+    );
+  }
+}

tsup.config.ts

@@ -1,7 +1,7 @@
 import { defineConfig } from "tsup";
 
 export default defineConfig({
-  entry: ["src/index.ts"],
+  entry: ["src/**/*.ts"],
   format: ["cjs", "esm"], // Build for commonJS and ESmodules
   dts: true, // Generate declaration file (.d.ts)
   splitting: false,