Lab 02 jev · typescript

jev()

Building

jev(accounts, "looks like a churn risk"). One call per row, a yes-or-no question, a probability back. No embeddings, no index, no prompt.

Type inside the quotes and press enter. jev.rank() asks for a score instead.
Rows resolve as their call returns and sort themselves. Click one to see what the model saw.
Threshold re-filters on screen. Repeating a query is cached and free.

See one call, exactly as the server sends it

POST /api/labs one row jev · × 48 rows

{
  "state": "Ana Ruiz at Lumen Labs (Madrid, Spain) · plan: team, 12 seats, $348/month · customer for 14 months · last seen 2 days ago · 9 sessions per week · 3 support tickets in the last 90 days · NPS 9 · last note: \"asked about SSO twice this month\"",
  "questions": {
    "match": {
      "type": "boolean",
      "instructions": "looks like a churn risk"
    }
  }
}

The state is one serialized row and match is the only question, so the answer is a single probability. jev.rank() swaps it for a score question with five levels and reads the expected value:

{
  "level": {
    "type": "score",
    "instructions": "how likely to add seats next quarter",
    "criteria": [
      "not at all",
      "slightly",
      "somewhat",
      "very",
      "extremely"
    ]
  }
}

The whole helper, 165 lines, no dependencies beyond the AI SDK

jev.ts filter · rank · classify

import { createGateway } from "@ai-sdk/gateway";
import {
  experimental_evaluate as evaluate,
  type Experimental_EvaluationQuestion,
  type Experimental_EvaluationResult,
} from "ai";

export interface Settled<T> {
  readonly item: T;
  readonly index: number;
  readonly latencyMs: number;
  readonly inputTokens: number;
  readonly cached: boolean;
}

export interface Match<T> extends Settled<T> {
  readonly p: number;
}

export interface Ranked<T> extends Settled<T> {
  readonly score: number;
}

export interface Labeled<T, L extends string> extends Settled<T> {
  readonly label: L;
  readonly p: number;
}

export interface JevOptions<T, R = Settled<T>> {
  readonly threshold?: number;
  readonly serialize?: (item: T) => string;
  readonly concurrency?: number;
  readonly signal?: AbortSignal | undefined;
  readonly apiKey?: string | undefined;
  readonly timeoutMs?: number;
  readonly onResult?: (result: R) => void;
  readonly onError?: (item: T, error: unknown) => void;
}

type Questions = Record<string, Experimental_EvaluationQuestion>;
type Answers<Q extends Questions> = Experimental_EvaluationResult<Q>["answers"];

interface Asked<Q extends Questions> {
  readonly answers: Answers<Q>;
  readonly latencyMs: number;
  readonly inputTokens: number;
  readonly cached: boolean;
}

export const jevModelId = "typesafe-ai/jev";
export const jevTimeoutMs = 4000;
export const rankLevels = ["not at all", "slightly", "somewhat", "very", "extremely"] as const;

const cache = new Map<string, { answers: unknown; inputTokens: number }>();
const cacheLimit = 5000;

function remember(key: string, answers: unknown, inputTokens: number): void {
  if (cache.size >= cacheLimit) cache.delete(cache.keys().next().value as string);
  cache.set(key, { answers, inputTokens });
}

function serializeByDefault(item: unknown): string {
  return typeof item === "string" ? item : JSON.stringify(item);
}

function deadline(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
  const timeout = AbortSignal.timeout(timeoutMs);
  return signal ? AbortSignal.any([signal, timeout]) : timeout;
}

async function ask<Q extends Questions>(
  state: string,
  questions: Q,
  options: Pick<JevOptions<unknown>, "signal" | "apiKey" | "timeoutMs">,
): Promise<Asked<Q>> {
  const key = `${JSON.stringify(questions)}\n${state}`;
  const hit = cache.get(key);
  if (hit) return { answers: hit.answers as Answers<Q>, latencyMs: 0, inputTokens: 0, cached: true };

  const gateway = createGateway(options.apiKey ? { apiKey: options.apiKey } : {});
  const startedAt = performance.now();
  const attempt = () =>
    evaluate({
      model: gateway.evaluationModel(jevModelId),
      state,
      questions,
      maxRetries: 0,
      abortSignal: deadline(options.signal, options.timeoutMs ?? jevTimeoutMs),
    });

  let result: Awaited<ReturnType<typeof attempt>>;
  try {
    result = await attempt();
  } catch (error) {
    if (options.signal?.aborted) throw error;
    result = await attempt();
  }

  const inputTokens = result.usage.inputTokens ?? 0;
  remember(key, result.answers, inputTokens);
  return { answers: result.answers, latencyMs: Math.round(performance.now() - startedAt), inputTokens, cached: false };
}

async function fanOut<T, Q extends Questions, R extends Settled<T>>(
  items: readonly T[],
  questions: Q,
  options: JevOptions<T, R>,
  settle: (base: Settled<T>, answers: Answers<Q>) => R,
): Promise<R[]> {
  const serialize = options.serialize ?? serializeByDefault;
  const results: R[] = [];
  let cursor = 0;

  async function work(): Promise<void> {
    while (cursor < items.length) {
      if (options.signal?.aborted) return;
      const index = cursor++;
      const item = items[index] as T;
      let asked: Asked<Q>;
      try {
        asked = await ask(serialize(item), questions, options);
      } catch (error) {
        if (options.signal?.aborted || !options.onError) throw error;
        options.onError(item, error);
        continue;
      }
      const result = settle({ item, index, latencyMs: asked.latencyMs, inputTokens: asked.inputTokens, cached: asked.cached }, asked.answers);
      results[index] = result;
      options.onResult?.(result);
    }
  }

  const workers = Math.max(1, Math.min(options.concurrency ?? 8, items.length));
  await Promise.all(Array.from({ length: workers }, work));
  return results.filter((result): result is R => result !== undefined);
}

export async function jev<T>(items: readonly T[], predicate: string, options: JevOptions<T, Match<T>> = {}): Promise<Match<T>[]> {
  const threshold = options.threshold ?? 0.5;
  const questions = { match: { type: "boolean", instructions: predicate } } as const satisfies Questions;
  const matches = await fanOut(items, questions, options, (base, answers) => ({ ...base, p: answers.match.probability }));
  return matches.filter((match) => match.p >= threshold).sort((a, b) => b.p - a.p);
}

jev.rank = async function rank<T>(items: readonly T[], criterion: string, options: JevOptions<T, Ranked<T>> = {}): Promise<Ranked<T>[]> {
  const questions = { level: { type: "score", instructions: criterion, criteria: [...rankLevels] } } as const satisfies Questions;
  const top = rankLevels.length - 1;
  const ranked = await fanOut(items, questions, options, (base, answers) => ({ ...base, score: Math.min(1, Math.max(0, answers.level.score / top)) }));
  return ranked.sort((a, b) => b.score - a.score);
};

jev.classify = async function classify<T, L extends string>(
  items: readonly T[],
  instructions: string,
  labels: Record<L, string>,
  options: JevOptions<T, Labeled<T, L>> = {},
): Promise<Labeled<T, L>[]> {
  const questions = { label: { type: "choice", instructions, criteria: labels } } as const satisfies Questions;
  return fanOut(items, questions, options, (base, answers) => {
    const label = answers.label.choice as L;
    const probabilities = (answers.label.probabilities ?? {}) as Partial<Record<L, number>>;
    return { ...base, label, p: probabilities[label] ?? 1 };
  });
};

Limited concurrency, a serializer you control, an in-memory cache keyed by row and question, and an AbortSignal so a new predicate cancels the previous fan-out. Drop it in any TypeScript project with an AI Gateway key.