kiln_ai.datamodel.eval

   1import json
   2import math
   3from enum import Enum
   4from threading import Lock
   5from typing import TYPE_CHECKING, Annotated, Any, Dict, List, Literal, Union
   6
   7from pydantic import (
   8    BaseModel,
   9    ConfigDict,
  10    Discriminator,
  11    Field,
  12    GetJsonSchemaHandler,
  13    JsonValue,
  14    SerializationInfo,
  15    SerializerFunctionWrapHandler,
  16    TypeAdapter,
  17    ValidationInfo,
  18    model_serializer,
  19    model_validator,
  20)
  21from pydantic.json_schema import JsonSchemaValue
  22from pydantic_core import CoreSchema
  23from typing_extensions import Self
  24
  25from kiln_ai.datamodel.basemodel import (
  26    ID_TYPE,
  27    FilenameString,
  28    FilenameStringShort,
  29    KilnParentedModel,
  30    KilnParentModel,
  31)
  32from kiln_ai.datamodel.code_file_storage import (
  33    read_code_from_sibling_file,
  34    write_code_to_sibling_file,
  35)
  36from kiln_ai.datamodel.datamodel_enums import (
  37    EvalStatus,
  38    Priority,
  39    TaskOutputRatingType,
  40)
  41from kiln_ai.datamodel.dataset_filters import DatasetFilterId, EvalInputFilterId
  42from kiln_ai.datamodel.json_schema import string_to_json_key
  43from kiln_ai.datamodel.task_run import Usage
  44from kiln_ai.datamodel.tool_id import ToolId, validate_tool_allowlist
  45from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
  46
  47if TYPE_CHECKING:
  48    from kiln_ai.datamodel.spec import Spec
  49    from kiln_ai.datamodel.task import Task
  50    from kiln_ai.datamodel.task_run import TaskRun
  51
  52EvalScores = Dict[str, float]
  53
  54# Module-level set to track evals currently being migrated (to prevent recursion)
  55# Protected by _migration_lock to ensure thread-safe access
  56_migration_lock = Lock()
  57_currently_migrating_eval_ids: set[ID_TYPE] = set()
  58
  59# Fixed name of the sibling file that holds a code judge's Python source, stored
  60# beside its eval_config.kiln. Fixed so authored tests can `from scorer import score`.
  61SCORER_CODE_FILENAME = "scorer.py"
  62
  63
  64class EvalTemplateId(str, Enum):
  65    """
  66    An eval template is a pre-defined eval that can be used as a starting point for a new eval.
  67    """
  68
  69    kiln_requirements = "kiln_requirements"
  70    desired_behaviour = "desired_behaviour"
  71    issue = "kiln_issue"
  72    tool_call = "tool_call"
  73    toxicity = "toxicity"
  74    bias = "bias"
  75    maliciousness = "maliciousness"
  76    factual_correctness = "factual_correctness"
  77    jailbreak = "jailbreak"
  78    rag = "rag"
  79
  80
  81class EvalConfigType(str, Enum):
  82    """The type of eval configuration, determining how scores are generated."""
  83
  84    g_eval = "g_eval"
  85    llm_as_judge = "llm_as_judge"
  86    v2 = "v2"
  87
  88
  89class V2EvalType(str, Enum):
  90    """V2-only eval type enum. Each value maps to a typed properties class
  91    and a V2 adapter."""
  92
  93    llm_judge = "llm_judge"
  94    exact_match = "exact_match"
  95    pattern_match = "pattern_match"
  96    set_check = "set_check"
  97    tool_call_check = "tool_call_check"
  98    contains = "contains"
  99    step_count_check = "step_count_check"
 100    code_eval = "code_eval"
 101
 102
 103class LlmJudgeProperties(BaseModel):
 104    type: Literal[V2EvalType.llm_judge] = V2EvalType.llm_judge
 105    model_name: str
 106    model_provider: str
 107    system_prompt: str | None = None
 108    prompt_template: str
 109    reference_keys: list[str] = []
 110    thinking_instruction: str | None = None
 111    g_eval: bool = False
 112    # User-written evaluation steps, bound to {{ judge_instructions }} when the
 113    # prompt template is rendered. Used by evals with no spec or template to
 114    # derive default steps from.
 115    judge_instructions: list[str] | None = None
 116
 117
 118class ExactMatchProperties(BaseModel):
 119    type: Literal[V2EvalType.exact_match] = V2EvalType.exact_match
 120    value_expression: str | None = None
 121    expected_value: str | None = None
 122    reference_key: str | None = Field(default=None, min_length=1)
 123    case_sensitive: bool = True
 124
 125    @model_validator(mode="after")
 126    def validate_value_source(self) -> Self:
 127        if (self.expected_value is None) == (self.reference_key is None):
 128            raise ValueError(
 129                "Exactly one of expected_value or reference_key must be set"
 130            )
 131        return self
 132
 133
 134class PatternMatchProperties(BaseModel):
 135    type: Literal[V2EvalType.pattern_match] = V2EvalType.pattern_match
 136    value_expression: str | None = None
 137    pattern: str
 138    mode: Literal["must_match", "must_not_match"] = "must_match"
 139
 140    @model_validator(mode="after")
 141    def validate_pattern(self) -> Self:
 142        import re
 143
 144        try:
 145            re.compile(self.pattern)
 146        except re.error as e:
 147            raise ValueError(f"Invalid regex pattern '{self.pattern}': {e}") from e
 148        return self
 149
 150
 151class ContainsProperties(BaseModel):
 152    type: Literal[V2EvalType.contains] = V2EvalType.contains
 153    value_expression: str | None = None
 154    substring: str | None = None
 155    reference_key: str | None = Field(default=None, min_length=1)
 156    case_sensitive: bool = True
 157    mode: Literal["must_contain", "must_not_contain"] = "must_contain"
 158
 159    @model_validator(mode="after")
 160    def validate_value_source(self) -> Self:
 161        if (self.substring is None) == (self.reference_key is None):
 162            raise ValueError("Exactly one of substring or reference_key must be set")
 163        return self
 164
 165
 166class SetCheckProperties(BaseModel):
 167    type: Literal[V2EvalType.set_check] = V2EvalType.set_check
 168    value_expression: str | None = None
 169    expected_set: list[str] | None = None
 170    reference_key: str | None = Field(default=None, min_length=1)
 171    mode: Literal["subset", "superset", "equal"]
 172
 173    @model_validator(mode="after")
 174    def validate_value_source(self) -> Self:
 175        if (self.expected_set is None) == (self.reference_key is None):
 176            raise ValueError("Exactly one of expected_set or reference_key must be set")
 177        return self
 178
 179
 180class ArgMatch(BaseModel):
 181    value: JsonValue
 182    match_mode: Literal["exact", "contains", "regex"] = "exact"
 183
 184    @model_validator(mode="after")
 185    def validate_regex(self) -> Self:
 186        if self.match_mode == "regex":
 187            import re
 188
 189            try:
 190                re.compile(str(self.value))
 191            except re.error as e:
 192                raise ValueError(f"Invalid regex value '{self.value}': {e}") from e
 193        return self
 194
 195
 196class ToolCallSpec(BaseModel):
 197    tool_name: str
 198    expected_args: dict[str, ArgMatch] | None = None
 199
 200
 201class ToolCallCheckProperties(BaseModel):
 202    type: Literal[V2EvalType.tool_call_check] = V2EvalType.tool_call_check
 203    expected_tools: list[ToolCallSpec] = Field(min_length=1)
 204    match_mode: Literal["any", "all", "ordered", "never"] = "all"
 205    on_unexpected_tools: Literal["ignore", "fail"] = "ignore"
 206
 207
 208class StepCountCheckProperties(BaseModel):
 209    type: Literal[V2EvalType.step_count_check] = V2EvalType.step_count_check
 210    count_type: Literal["tool_calls", "model_responses", "turns"]
 211    min_count: int | None = None
 212    max_count: int | None = None
 213
 214    @model_validator(mode="after")
 215    def check_bounds(self) -> Self:
 216        if self.min_count is None and self.max_count is None:
 217            raise ValueError(
 218                "step_count_check requires at least one of min_count / max_count"
 219            )
 220        if (
 221            self.min_count is not None
 222            and self.max_count is not None
 223            and self.min_count > self.max_count
 224        ):
 225            raise ValueError("min_count must be <= max_count")
 226        return self
 227
 228
 229class CodeEvalProperties(BaseModel):
 230    type: Literal[V2EvalType.code_eval] = V2EvalType.code_eval
 231    code: str
 232    reference_keys: list[str] = []
 233    timeout_seconds: int = Field(default=180, ge=1, le=300)
 234    tool_allowlist: list[ToolId] = Field(
 235        default_factory=list,
 236        description="Explicit per-tool allowlist of tools the scorer code may call.",
 237    )
 238
 239    @model_validator(mode="after")
 240    def validate_allowlist(self) -> Self:
 241        # No self-reference check: a code eval is not itself a tool.
 242        validate_tool_allowlist(self.tool_allowlist, caller="code evals")
 243        return self
 244
 245    @model_validator(mode="before")
 246    @classmethod
 247    def _read_code_file(cls, data: Any, info: ValidationInfo) -> Any:
 248        """When loading from disk, inject `code` from the sibling scorer.py.
 249
 250        The source is stored in scorer.py beside eval_config.kiln, not inline in
 251        the JSON. CodeEvalProperties is a nested member of the
 252        V2EvalConfigProperties discriminated union in EvalConfig.properties, so
 253        the load context set on the parent EvalConfig (`source_dir`) propagates
 254        down to this validator. The shared helper reads the file here, before
 255        field validation, so the existing validate_code trio runs against the
 256        loaded string unchanged.
 257        """
 258        # Explicit type-gate (defense-in-depth): this validator only ever runs
 259        # for code_eval properties — it lives on CodeEvalProperties, and both the
 260        # discriminated union and the eager parse route only code_eval dicts
 261        # here. Assert that gate so a future refactor can't quietly read
 262        # scorer.py for another eval type. None (type omitted, field defaults)
 263        # and the enum form both pass; only a present, mismatched type is
 264        # rejected, so valid-input behavior is unchanged.
 265        if isinstance(data, dict) and data.get("type") not in (
 266            None,
 267            V2EvalType.code_eval.value,
 268        ):
 269            raise ValueError(
 270                "CodeEvalProperties can only load code_eval properties, "
 271                f"got type: {data.get('type')!r}"
 272            )
 273        return read_code_from_sibling_file(
 274            data,
 275            info.context or {},
 276            filename=SCORER_CODE_FILENAME,
 277            kiln_filename="eval_config.kiln",
 278            model_label="CodeEvalProperties",
 279        )
 280
 281    @model_serializer(mode="wrap")
 282    def _serialize(
 283        self, handler: SerializerFunctionWrapHandler, info: SerializationInfo
 284    ) -> dict[str, Any]:
 285        """On disk-save, write `code` to scorer.py and omit it from the .kiln JSON.
 286
 287        Delegates to the shared sibling-file helper, which uses the same save
 288        context attachments use (`save_attachments` + `dest_path`); it propagates
 289        from the parent EvalConfig's save_to_file() down to this nested union
 290        member. Without that context — normal model_dump / API responses —
 291        `code` is left in the output and no file is written, so the API contract
 292        is unchanged. The default handler preserves `type` (needed by the
 293        discriminator), `reference_keys`, and `timeout_seconds`.
 294
 295        Schema note: a custom model_serializer would otherwise collapse the
 296        *serialization-mode* JSON schema to an untyped object
 297        (`model_json_schema(mode="serialization")` loses per-field typing).
 298        Unlike CodeTool — which is never a FastAPI response_model — this model is
 299        nested in EvalConfig, and EvalConfig IS the declared `response_model` on
 300        several endpoints (eval_api.py). FastAPI generates response schemas in
 301        serialization mode, so a collapsed schema here would split
 302        CodeEvalProperties into an untyped `-Output` component and drift the
 303        checked-in api_schema.d.ts (breaking check_schema.sh and the web types
 304        that key off `components["schemas"]["CodeEvalProperties"]`). The
 305        `__get_pydantic_json_schema__` override below is therefore REQUIRED (not
 306        optional): it keeps the serialization-mode schema identical to
 307        validation mode. Do not remove either the serializer (runtime file
 308        storage) or the override (schema stability).
 309        """
 310        return write_code_to_sibling_file(
 311            handler(self),
 312            info.context or {},
 313            filename=SCORER_CODE_FILENAME,
 314            code=self.code,
 315        )
 316
 317    @classmethod
 318    def __get_pydantic_json_schema__(
 319        cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler
 320    ) -> JsonSchemaValue:
 321        """Keep the serialization-mode JSON schema identical to validation mode.
 322
 323        The wrap serializer above returns an untyped `dict`, which would collapse
 324        this model's serialization-mode JSON schema to `{additionalProperties:
 325        true, type: object}` (dropping `code`, `type`, etc.). Because EvalConfig
 326        (which nests this model) is a FastAPI response_model, that collapse would
 327        drift the committed OpenAPI/api_schema.d.ts. Dropping the `serialization`
 328        core-schema entries makes JSON-schema generation use the field-based
 329        (validation) representation in both modes, so `code` stays present and
 330        typed and there is no `-Input`/`-Output` split. The custom serializer
 331        lives on the inner `model` core schema (the before/after validators wrap
 332        it in function schemas), so the strip must be recursive. This affects
 333        only schema generation, never runtime (de)serialization.
 334        """
 335
 336        def strip_serialization(schema: Any) -> Any:
 337            if isinstance(schema, dict):
 338                return {
 339                    key: strip_serialization(value)
 340                    for key, value in schema.items()
 341                    if key != "serialization"
 342                }
 343            if isinstance(schema, list):
 344                return [strip_serialization(item) for item in schema]
 345            return schema
 346
 347        return handler(strip_serialization(core_schema))
 348
 349    @model_validator(mode="after")
 350    def validate_code(self) -> Self:
 351        code_bytes = self.code.encode("utf-8")
 352        if len(code_bytes) > 64 * 1024:
 353            raise ValueError(
 354                f"Code is too large ({len(code_bytes)} bytes). Maximum size is 64KB."
 355            )
 356
 357        try:
 358            compile(self.code, "<code_eval>", "exec")
 359        except SyntaxError as e:
 360            raise ValueError(f"Code has a syntax error: {e}") from e
 361
 362        import ast
 363
 364        tree = ast.parse(self.code)
 365        # Both sync and async score functions are accepted here.
 366        # Async coroutines are transparently awaited in sandbox_worker.execute_scorer_bridged.
 367        has_score_fn = any(
 368            isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
 369            and node.name == "score"
 370            for node in ast.iter_child_nodes(tree)
 371        )
 372        if not has_score_fn:
 373            raise ValueError(
 374                "Code must define a module-level 'score' function (def score(...))."
 375            )
 376
 377        return self
 378
 379
 380V2EvalConfigProperties = Annotated[
 381    Union[
 382        LlmJudgeProperties,
 383        ExactMatchProperties,
 384        PatternMatchProperties,
 385        SetCheckProperties,
 386        ToolCallCheckProperties,
 387        ContainsProperties,
 388        StepCountCheckProperties,
 389        CodeEvalProperties,
 390    ],
 391    Discriminator("type"),
 392]
 393
 394# Parses a raw properties dict into its typed V2 class via the "type" discriminator.
 395_V2_PROPERTIES_ADAPTER: TypeAdapter[Any] = TypeAdapter(V2EvalConfigProperties)
 396
 397# Explicit tuple of V2 property types for isinstance() checks.
 398# Must list exactly the same types as the V2EvalConfigProperties union above.
 399V2_PROPERTY_TYPES: tuple[type[BaseModel], ...] = (
 400    LlmJudgeProperties,
 401    ExactMatchProperties,
 402    PatternMatchProperties,
 403    SetCheckProperties,
 404    ToolCallCheckProperties,
 405    ContainsProperties,
 406    StepCountCheckProperties,
 407    CodeEvalProperties,
 408)
 409
 410
 411def reference_data_keys(props: V2EvalConfigProperties) -> list[str]:
 412    """Return the reference-data keys a single judge needs.
 413
 414    Exhaustive match over the V2 properties union: adding a new V2 type
 415    without handling it here will fail ``ty`` type-checking.
 416    """
 417    match props:
 418        case ExactMatchProperties():
 419            return [props.reference_key] if props.reference_key else []
 420        case ContainsProperties():
 421            return [props.reference_key] if props.reference_key else []
 422        case SetCheckProperties():
 423            return [props.reference_key] if props.reference_key else []
 424        case LlmJudgeProperties():
 425            return list(props.reference_keys)
 426        case CodeEvalProperties():
 427            return list(props.reference_keys)
 428        case PatternMatchProperties():
 429            return []
 430        case ToolCallCheckProperties():
 431            return []
 432        case StepCountCheckProperties():
 433            return []
 434        case _:
 435            raise_exhaustive_enum_error(props)
 436
 437
 438def _eager_parse_code_eval_on_load(
 439    data: dict[str, Any], ctx: dict[str, Any]
 440) -> dict[str, Any]:
 441    """Eagerly parse a code_eval EvalConfig's `properties` on file load.
 442
 443    V2 code judges store their score() source in a sibling scorer.py, not inline
 444    in the JSON. On load, parse a code_eval properties dict through
 445    CodeEvalProperties (which reads scorer.py via the propagated load context) so
 446    any error surfaces directly. Without this, the outer
 447    `V2EvalConfigProperties | dict | None` union would recover from the nested
 448    member's error by falling back to the dict branch, masking the real cause
 449    (e.g. a missing scorer.py or a bad score() function) behind a generic
 450    "V2 config requires typed properties". See functional spec §2.2 / §4.
 451
 452    Only touches code_eval properties during a file load, gated explicitly on
 453    `type == code_eval`; every other input passes through unchanged. Lifted
 454    verbatim from EvalConfig.dispatch_properties_parsing so the code-eval load
 455    path is a clearly-named, code-eval-local step rather than smeared into the
 456    generic dispatcher.
 457    """
 458    if not ctx.get("loading_from_file"):
 459        return data
 460    props = data.get("properties")
 461    if isinstance(props, dict) and props.get("type") == V2EvalType.code_eval.value:
 462        data = dict(data)
 463        data["properties"] = CodeEvalProperties.model_validate(props, context=ctx)
 464    return data
 465
 466
 467def validate_scores_against_output_scores(
 468    scores: EvalScores,
 469    output_scores: list["EvalOutputScore"],
 470) -> list[str]:
 471    """Validate that *scores* fall within the expected range for each output score.
 472
 473    Returns a list of human-readable problem strings (empty list means all OK).
 474    This is a pure function — it does NOT raise; callers decide how to surface errors.
 475    """
 476
 477    def _is_numeric(v: object) -> bool:
 478        # NaN compares False against every range bound, so it passes every check
 479        # below, then serializes to null and makes the saved file fail on reload.
 480        if not isinstance(v, (int, float)) or isinstance(v, bool):
 481            return False
 482        try:
 483            return math.isfinite(v)
 484        except OverflowError:
 485            # isfinite coerces int args to float, which an int like 10**400 can't be.
 486            return False
 487
 488    problems: list[str] = []
 489    for output_score in output_scores:
 490        key = output_score.json_key()
 491        if key not in scores:
 492            continue
 493        value = scores[key]
 494
 495        match output_score.type:
 496            case TaskOutputRatingType.five_star:
 497                if not _is_numeric(value) or value < 1.0 or value > 5.0:
 498                    problems.append(
 499                        f"Score {output_score.name} is a five_star rating and must be a number between 1.0 and 5.0 inclusive. Got: {value}"
 500                    )
 501            case TaskOutputRatingType.pass_fail:
 502                if not _is_numeric(value) or value < 0.0 or value > 1.0:
 503                    problems.append(
 504                        f"Score {output_score.name} is a pass_fail rating and must be a number between 0.0 and 1.0 inclusive. Got: {value}"
 505                    )
 506            case TaskOutputRatingType.pass_fail_critical:
 507                if not _is_numeric(value) or value < -1.0 or value > 1.0:
 508                    problems.append(
 509                        f"Score {output_score.name} is a pass_fail_critical rating and must be a number between -1.0 and 1.0 inclusive. Got: {value}"
 510                    )
 511            case TaskOutputRatingType.custom:
 512                problems.append(
 513                    f"Custom scores are not supported in evaluators. '{output_score.name}' was set to a custom score."
 514                )
 515            case _:
 516                raise_exhaustive_enum_error(output_score.type)
 517    return problems
 518
 519
 520class SkippedReason(str, Enum):
 521    """Terminal skip reasons stored as str for back/forward-compat."""
 522
 523    missing_reference_key = "missing_reference_key"
 524    extraction_failed = "extraction_failed"
 525    missing_trace = "missing_trace"
 526    missing_drive_config = "missing_drive_config"
 527    incompatible_input_shape = "incompatible_input_shape"
 528    code_eval_not_trusted = "code_eval_not_trusted"
 529    type_not_available = "type_not_available"
 530
 531
 532class V2EvalResult(BaseModel):
 533    """Result of a single V2 eval ``evaluate()`` call."""
 534
 535    scores: EvalScores = Field(default_factory=dict)
 536    skipped_reason: SkippedReason | None = None
 537    skipped_detail: str | None = None
 538    intermediate_outputs: Dict[str, str] | None = None
 539    usage: Usage | None = Field(
 540        default=None,
 541        description="What the judgment itself cost, if it called a model. None for the deterministic eval types, which call none. Stored on the resulting EvalRun as eval_usage.",
 542    )
 543
 544
 545class UserMessage(BaseModel):
 546    text: str
 547
 548
 549class SyntheticUserInfo(BaseModel):
 550    """The synthetic user's character sheet: who they are and what they want.
 551
 552    This is both the persisted form on multi-turn synthetic eval inputs and
 553    the runtime shape the synthetic-user driver renders its system prompt
 554    from. The XML-tagged blob some wire formats carry is parsed into this at
 555    the wire boundary (kiln_ai.synthetic_user.parser) — it is never stored.
 556
 557    extra="allow": unknown fields from newer generators survive load/save
 558    round-trips instead of being dropped.
 559    """
 560
 561    model_config = ConfigDict(extra="allow")
 562
 563    persona: str
 564    goal: str
 565    behavior_guidance: str | None = None
 566
 567
 568class SingleTurnEvalInputData(BaseModel):
 569    type: Literal["single_turn"] = "single_turn"
 570    user_message: UserMessage
 571
 572
 573class MultiTurnDriveConfig(BaseModel):
 574    """Settings for re-driving a multi-turn synthetic input at eval time.
 575
 576    A multi-turn eval run regenerates each conversation: the agent under test
 577    comes from the run config being evaluated, while the synthetic user
 578    (customer) configured here is held constant across run configs — so a
 579    comparison varies only the agent. Stored per item, on
 580    MultiTurnSyntheticEvalInputData.drive_config.
 581    """
 582
 583    model_name: str = Field(
 584        description="The model that plays the synthetic user during re-drives."
 585    )
 586    # A plain string rather than the provider enum so persisted items load on
 587    # builds that don't know the provider yet (same choice as LlmJudgeProperties).
 588    model_provider: str = Field(description="The provider of the synthetic-user model.")
 589    turns: int = Field(
 590        ge=1,
 591        le=20,
 592        description="Ceiling on the assistant turns per re-driven conversation.",
 593    )
 594
 595
 596class MultiTurnSyntheticEvalInputData(BaseModel):
 597    """A re-drivable multi-turn case: the opening user message, the synthetic
 598    user who continues the conversation at eval time, and the drive settings
 599    that synthetic user runs with.
 600
 601    Together these make the item a self-contained replication recipe: with the
 602    persona, first_message, and drive_config it re-drives identically under any
 603    eval that references it, which is what makes conversation traces keyed to
 604    the item reusable across evals.
 605
 606    first_message may be None; such items carry no seed to open a
 607    conversation with, so the eval runner skips them instead of re-driving.
 608    """
 609
 610    type: Literal["multi_turn_synthetic"] = "multi_turn_synthetic"
 611    first_message: UserMessage | None = None
 612    synthetic_user_info: SyntheticUserInfo
 613    drive_config: MultiTurnDriveConfig | None = Field(
 614        default=None,
 615        description="How this item's conversation is re-driven: the "
 616        "synthetic-user model and turn count, stamped when the item is minted. "
 617        "This is the ONLY home for drive settings — no eval-level copy exists; "
 618        "displays and prefills derive from items. Held constant across run "
 619        "configs so a comparison varies only the agent under test. Immutable "
 620        "once minted: changing the synthetic-user setup means minting new "
 621        "items, which keeps traces keyed to this item valid. None only on "
 622        "items minted before drive settings were stamped; the eval runner "
 623        "skips such items with a clear reason rather than guessing a config.",
 624    )
 625
 626    @model_serializer(mode="wrap")
 627    def _omit_unset_drive_config(
 628        self, handler: SerializerFunctionWrapHandler
 629    ) -> Dict[str, Any]:
 630        # Items that predate drive_config carry no key on disk, and absent and
 631        # null load identically — omit an unset config instead of churning
 632        # every legacy file with a null on resave.
 633        data: Dict[str, Any] = handler(self)
 634        if data.get("drive_config") is None:
 635            data.pop("drive_config", None)
 636        return data
 637
 638
 639EvalInputData = Annotated[
 640    Union[
 641        SingleTurnEvalInputData,
 642        MultiTurnSyntheticEvalInputData,
 643    ],
 644    Discriminator("type"),
 645]
 646
 647
 648class EvalInput(KilnParentedModel):
 649    """A single evaluation input item, stored as a child of a Task.
 650
 651    Each EvalInput contains the data needed to run an evaluation (e.g. a user
 652    message) plus optional reference data for comparison and tags for filtering.
 653    """
 654
 655    data: EvalInputData = Field(
 656        description="The input data for this eval item.",
 657    )
 658    reference: dict[str, JsonValue] | None = Field(
 659        default=None,
 660        description="Optional reference data (ground truth) for this eval input, keyed by reference name.",
 661    )
 662    tags: list[str] = Field(
 663        default_factory=list,
 664        description="Tags for filtering eval inputs.",
 665    )
 666
 667    @model_validator(mode="after")
 668    def validate_tags(self) -> Self:
 669        # Empty or space-containing tags can't be selected by tag filters, so
 670        # reject them at creation instead of silently dropping the item later.
 671        for tag in self.tags:
 672            if not tag:
 673                raise ValueError("Tags cannot be empty strings")
 674            if " " in tag:
 675                raise ValueError("Tags cannot contain spaces. Try underscores.")
 676        return self
 677
 678
 679class EvalTaskInput(BaseModel):
 680    """The runtime data bundle passed to V2 evaluators.
 681
 682    Assembled by the eval runner from the item being evaluated and the task run that
 683    was scored. The item is either an EvalInput or a TaskRun drawn from the dataset;
 684    which one it is determines where `reference_data` and `task_input` come from.
 685    """
 686
 687    final_message: str = Field(
 688        description="The final model output (task output text).",
 689    )
 690    trace: list[dict[str, Any]] | None = Field(
 691        default=None,
 692        description="The full conversation trace, if available.",
 693    )
 694    reference_data: dict[str, JsonValue] | None = Field(
 695        default=None,
 696        description=(
 697            "Ground-truth data for the item being evaluated, keyed by reference name. "
 698            "Taken from EvalInput.reference for an EvalInput-backed item; for a "
 699            "TaskRun-backed dataset item it is the item's own stored output under the "
 700            "key 'reference_answer', since that output is the curated answer. None "
 701            "when a TaskRun is scored as itself (judge calibration), where the item "
 702            "and the scored run are the same record."
 703        ),
 704    )
 705    task_input: str | None = Field(
 706        default=None,
 707        description="The original task input text.",
 708    )
 709
 710    @classmethod
 711    def from_trace(
 712        cls, trace: "TaskRun", source: "TaskRun | EvalInput"
 713    ) -> "EvalTaskInput":
 714        """What a judge sees: the trace that was produced, plus the item it came from.
 715
 716        The two are separate arguments because they are separate records once eval traces
 717        live on their own TaskRun — the trace holds what the model said, and the source
 718        item holds the ground truth to compare it against. They are the same object only
 719        for calibration, where the golden dataset item is itself what gets scored.
 720        """
 721        from kiln_ai.datamodel.task_run import TaskRun as _TaskRun
 722
 723        if not isinstance(trace, _TaskRun):
 724            raise TypeError("Expected a TaskRun instance for trace")
 725
 726        trace_data: list[dict[str, Any]] | None = None
 727        if trace.trace is not None:
 728            trace_data = [dict(msg) for msg in trace.trace]
 729
 730        if isinstance(source, EvalInput):
 731            reference_data = source.reference
 732            # The item's own text, not the trace's: an EvalInput is the canonical
 733            # statement of the input, and the adapter may have reserialized it.
 734            if isinstance(source.data, SingleTurnEvalInputData):
 735                task_input = source.data.user_message.text
 736            elif isinstance(source.data, MultiTurnSyntheticEvalInputData):
 737                # Multi-turn: the first message opened the conversation, and the
 738                # rest of the exchange lives in the trace. Items minted without a
 739                # first message have no canonical input text to offer.
 740                task_input = (
 741                    source.data.first_message.text
 742                    if source.data.first_message
 743                    else None
 744                )
 745            else:
 746                raise ValueError(
 747                    f"Unsupported EvalInput data type: {type(source.data).__name__}"
 748                )
 749        elif isinstance(source, _TaskRun):
 750            # A TaskRun-backed dataset item stores the curated answer as its output, so
 751            # that output is the ground truth to compare the trace against. Skipped when
 752            # source *is* trace (calibration, and `from_task_run`): there the golden item
 753            # is itself what gets scored, so a reference would be byte-identical to
 754            # `final_message` and every judge comparing them would pass.
 755            reference_data = (
 756                None if source is trace else {"reference_answer": source.output.output}
 757            )
 758            task_input = trace.input
 759        else:
 760            raise TypeError("Expected a TaskRun or EvalInput instance for source")
 761
 762        return cls(
 763            final_message=trace.output.output,
 764            trace=trace_data,
 765            reference_data=reference_data,
 766            task_input=task_input,
 767        )
 768
 769    @classmethod
 770    def from_task_run(cls, task_run: "TaskRun") -> "EvalTaskInput":
 771        """A TaskRun scored as itself, with no separate source item."""
 772        return cls.from_trace(task_run, task_run)
 773
 774    @classmethod
 775    def from_eval_input(
 776        cls, eval_input: "EvalInput", run_output: "TaskRun"
 777    ) -> "EvalTaskInput":
 778        """A generated run scored against the EvalInput it was generated from.
 779
 780        The argument-order counterpart of `from_trace` for callers holding the item
 781        first. The explicit type check stays: passed a TaskRun by mistake,
 782        `from_trace` would silently take its TaskRun branch instead of failing.
 783        """
 784        if not isinstance(eval_input, EvalInput):
 785            raise TypeError("Expected an EvalInput instance")
 786        return cls.from_trace(run_output, eval_input)
 787
 788
 789class EvalOutputScore(BaseModel):
 790    """
 791    A definition of a score that an evaluator will produce.
 792
 793    Very similar to TaskRequirement, but conceptually different keeping in a separate models.
 794    """
 795
 796    name: FilenameStringShort = Field(
 797        description="The name of the score. Will be provided to the model so use a descriptive name. Should align to the model's TaskRequirement name if you want to use human evals to evaluate the evaluator's performance."
 798    )
 799    instruction: str | None = Field(
 800        default=None,
 801        description="A description of the score, used to help the model understand the goal of the score. Will be provided to evaluator models, so should be written for the model, not the team/user.",
 802    )
 803    type: TaskOutputRatingType = Field(
 804        description="The type of rating to use ('five_star', 'pass_fail', 'pass_fail_critical').",
 805    )
 806
 807    def json_key(self) -> str:
 808        """
 809        The JSON key for the score, used when running the evaluator with a LLM and we need JSON output.
 810
 811        For example, "Overall Rating" -> "overall_rating"
 812        """
 813        return string_to_json_key(self.name)
 814
 815    @model_validator(mode="after")
 816    def validate_type(self) -> Self:
 817        if self.type == TaskOutputRatingType.custom:
 818            raise ValueError(
 819                f"Custom scores are not supported in evaluators. Score '{self.name}' was set to a custom score."
 820            )
 821        return self
 822
 823
 824LEGACY_TRACE_FIELDS = ("output", "task_run_trace", "task_run_usage", "reference_answer")
 825"""The EvalRun fields that hold a copy of the trace a score was computed over.
 826
 827Deprecated: new records point at a TaskRun with `scored_run_id` instead. Kept declared
 828and loadable forever - every record written before the split still carries them. A
 829pointer-mode record must leave all of them None, which `validate_record_mode` enforces.
 830`input` is deprecated alongside these but is not in this tuple: it is the one a legacy
 831record is *required* to have, so it is checked separately.
 832
 833Three ways to mark a pydantic field deprecated; these fields use two of them:
 834
 8351. A `DEPRECATED:` prefix in the `description`. Reaches a human reading the SDK docs or
 836   the OpenAPI schema, and nothing else. Used.
 8372. `json_schema_extra={"deprecated": True}`. Puts `"deprecated": true` in the JSON
 838   schema, which `openapi-typescript` turns into a `/** @deprecated */` JSDoc tag, so
 839   the TS compiler and editors strike through every web call site. No runtime effect.
 840   Used.
 8413. `Field(deprecated=True)`. Same schema output as (2), but pydantic also raises a
 842   DeprecationWarning on every attribute *read* — and reading these is the correct,
 843   permanent way to render a legacy record, so it would be a warning storm. Not used.
 844   Do not "fix" this to (3) without silencing that first; (2) already provides the
 845   tooling signal (3) would be reached for."""
 846
 847
 848class EvalRun(KilnParentedModel):
 849    """
 850    The scores an eval produced for a single dataset item.
 851
 852    A run serves one of two purposes:
 853    - eval_config_eval=False (scoring): evaluating a task run config — the item's
 854      input was run through the task with task_run_config_id (which must be set)
 855      and the evaluator scored that output.
 856    - eval_config_eval=True (calibration): evaluating the eval config itself — an
 857      existing human-rated dataset item's input and output were scored so the
 858      evaluator can be compared against those human ratings. task_run_config_id
 859      must be None.
 860
 861    A record is described by two independent facts — whether it points at a TaskRun, and
 862    whether it was skipped — which `validate_record_mode` constrains to three legal
 863    shapes. What is exclusive is where the trace lives: on the record, or on the TaskRun,
 864    never both.
 865
 866    - **Pointer** (new): `scored_run_id` names the TaskRun that holds the trace. All
 867      inline trace fields must be None.
 868    - **Skipped**: `skipped_reason` set, so scores are not required. It also carries a
 869      `scored_run_id` if the trace existed and only scoring was skipped — so a skip can
 870      be a pointer record too — and none if the skip happened before generation.
 871    - **Legacy inline**: no `scored_run_id`; the trace lives on this record, and `input`
 872      is required unless the record was skipped. Every record written before the
 873      trace/score split is in this state, and it stays valid forever.
 874    """
 875
 876    dataset_id: ID_TYPE | None = Field(
 877        default=None,
 878        description="The ID of the dataset item (TaskRun) that was used for this run. Mutually exclusive with eval_input_id.",
 879    )
 880    scored_run_id: ID_TYPE | None = Field(
 881        default=None,
 882        description="The ID of the TaskRun this score was computed over. None for legacy records that carry their trace inline. A dangling reference is tolerated: the score still renders and still aggregates, only the trace drill-through is unavailable.",
 883    )
 884    task_run_config_id: ID_TYPE | None = Field(
 885        description="The ID of the TaskRunConfig that was run, if this eval run was based on a task run. Must belong to the same Task as this eval. Can be None if this eval run is based on an eval config."
 886    )
 887    eval_config_eval: bool = Field(
 888        description="Whether this eval run to evaluate the parent eval config (evaluating the config using an existing dataset item). If true, task_run_config_id must be None, as we're not running the task.",
 889        default=False,
 890    )
 891    input: str | None = Field(
 892        default=None,
 893        json_schema_extra={"deprecated": True},
 894        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.input instead. The input to the task. JSON formatted for structured input, plaintext for unstructured input. Required on legacy records (those with neither a scored_run_id nor a skipped_reason), never set on new ones.",
 895    )
 896    output: str | None = Field(
 897        default=None,
 898        json_schema_extra={"deprecated": True},
 899        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.output.output instead. The output of the task. None for skipped-before-execution runs.",
 900    )
 901    reference_answer: str | None = Field(
 902        default=None,
 903        json_schema_extra={"deprecated": True},
 904        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id. The reference answer for the input. JSON formatted for structured reference answer, plaintext for unstructured reference answer. Used for reference answer evals.",
 905    )
 906    intermediate_outputs: Dict[str, str] | None = Field(
 907        default=None,
 908        description="The intermediate outputs of the task (example, eval thinking).",
 909    )
 910    task_run_trace: str | None = Field(
 911        default=None,
 912        json_schema_extra={"deprecated": True},
 913        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.trace instead. The JSON formatted trace of the task run that produced the output.",
 914    )
 915    scores: EvalScores = Field(
 916        default={},
 917        description="The output scores of the evaluator (aligning to those required by the grand-parent Eval this object is a child of).",
 918    )
 919    task_run_usage: Usage | None = Field(
 920        default=None,
 921        json_schema_extra={"deprecated": True},
 922        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.usage instead. The usage of the task run that produced this eval run output (not the usage by the evaluation model).",
 923    )
 924    eval_usage: Usage | None = Field(
 925        default=None,
 926        description="The usage of the evaluation model (judge) that produced this eval run's scores, aggregated across every LLM call the judgment made. Distinct from task_run_usage, which is the evaluated task run's usage. None for non-LLM evals (e.g. code evals) and for records that predate this field.",
 927    )
 928
 929    eval_input_id: ID_TYPE | None = Field(
 930        default=None,
 931        description="ID of the EvalInput used for this run (V2 evals). Mutually exclusive with dataset_id.",
 932    )
 933    skipped_reason: str | None = Field(
 934        default=None,
 935        description="If set, this run was skipped. Stored as str for back/forward-compat; conventionally a SkippedReason value.",
 936    )
 937    skipped_detail: str | None = Field(
 938        default=None,
 939        description="Case-specific detail for skipped runs (e.g. missing key name).",
 940    )
 941
 942    def parent_eval_config(self) -> Union["EvalConfig", None]:
 943        if self.parent is not None and self.parent.__class__.__name__ != "EvalConfig":
 944            raise ValueError("parent must be an EvalConfig")
 945        return self.parent  # type: ignore
 946
 947    @model_validator(mode="after")
 948    def validate_input_source(self) -> Self:
 949        if (self.dataset_id is None) == (self.eval_input_id is None):
 950            raise ValueError(
 951                "Exactly one of dataset_id (V1 TaskRun source) or "
 952                "eval_input_id (V2 EvalInput source) must be set"
 953            )
 954        return self
 955
 956    @model_validator(mode="after")
 957    def validate_record_mode(self) -> Self:
 958        """Keep the three record states (pointer / skipped / legacy inline) exclusive.
 959
 960        The forbidding half of the pointer rule is the one that earns its keep: a record
 961        that points at a TaskRun must never also carry a second copy of what it scored,
 962        which nothing would keep in sync.
 963        """
 964        inline_set = [f for f in LEGACY_TRACE_FIELDS if getattr(self, f) is not None]
 965
 966        if self.scored_run_id is not None:
 967            # Checked before the skip branch on purpose: a record skipped at *scoring*
 968            # time still has a scored_run_id, and still must not carry inline data.
 969            if self.input is not None or inline_set:
 970                carried = (["input"] if self.input is not None else []) + inline_set
 971                raise ValueError(
 972                    "An EvalRun with scored_run_id must not carry inline trace data "
 973                    f"(set: {', '.join(carried)}). "
 974                    "The trace lives on the referenced TaskRun."
 975                )
 976            return self
 977
 978        if self.skipped_reason is not None:
 979            # Skipped before generation: there is nothing to point at, and nothing to
 980            # require. Legacy skipped records that do carry inline data stay valid.
 981            return self
 982
 983        if self.input is None:
 984            raise ValueError("A legacy EvalRun (no scored_run_id) requires input.")
 985        return self
 986
 987    @model_validator(mode="after")
 988    def validate_output_fields(self, info: ValidationInfo) -> Self:
 989        parent_eval_config = self.parent_eval_config()
 990        if self.scored_run_id is not None:
 991            # Pointer mode: the trace lives on the referenced TaskRun, and
 992            # validate_record_mode already forbids inline copies here. The checks
 993            # below are about data carried on this record, so none of them apply.
 994            return self
 995        parent_eval = parent_eval_config.parent_eval() if parent_eval_config else None
 996        if not parent_eval:
 997            return self
 998
 999        evaluation_data_type = parent_eval.evaluation_data_type
1000
1001        # A full_trace eval scores the conversation trace, so a successful task
1002        # run must carry it. Both V1 and V2 writers attach the trace for exactly
1003        # this shape (a scored, non-skipped task-run eval of a full_trace eval),
1004        # so demanding it back makes a writer that drops the trace fail loudly
1005        # instead of persisting a record that can't be re-scored. Historical
1006        # files predating this gate are exempt so they still load; new writes
1007        # and rebuilds are held to it.
1008        if (
1009            not self.eval_config_eval
1010            and self.skipped_reason is None
1011            and evaluation_data_type == EvalDataType.full_trace
1012            and self.task_run_trace is None
1013            and not self.loaded_from_file(info)
1014        ):
1015            raise ValueError("full_trace task run eval runs should include trace")
1016
1017        # Remaining checks are V1-only. V2 deliberately relaxes them: skipped
1018        # runs carry no output, and V2 writers never attach a trace to a
1019        # final_answer run in the first place.
1020        if parent_eval_config.config_type == EvalConfigType.v2:
1021            return self
1022
1023        if self.output is None and self.skipped_reason is None:
1024            raise ValueError("V1 EvalRun requires output to be set")
1025
1026        if (
1027            evaluation_data_type == EvalDataType.final_answer
1028            and self.task_run_trace is not None
1029        ):
1030            raise ValueError("final_answer runs should not set trace")
1031
1032        return self
1033
1034    @model_validator(mode="after")
1035    def validate_eval_run_types(self) -> Self:
1036        if self.eval_config_eval and self.task_run_config_id is not None:
1037            raise ValueError(
1038                "task_run_config_id must be None if eval_config_eval is true"
1039            )
1040        if not self.eval_config_eval and self.task_run_config_id is None:
1041            raise ValueError(
1042                "task_run_config_id must be set if eval_config_eval is false"
1043            )
1044        if self.eval_config_eval and self.dataset_id is None:
1045            raise ValueError(
1046                "eval_config_eval records must score a dataset item: judge "
1047                "calibration compares against human ratings, which only "
1048                "dataset items (TaskRuns) carry"
1049            )
1050        return self
1051
1052    @model_validator(mode="after")
1053    def validate_scores(self) -> Self:
1054        if self.skipped_reason is not None:
1055            return self
1056
1057        if self.scores is None or len(self.scores) == 0:
1058            raise ValueError("scores are required, and must have at least one score.")
1059
1060        parent_eval_config = self.parent_eval_config()
1061        eval = parent_eval_config.parent_eval() if parent_eval_config else None
1062        if not eval:
1063            return self
1064
1065        output_score_keys = [score.json_key() for score in eval.output_scores]
1066        if set(output_score_keys) != set(self.scores.keys()):
1067            raise ValueError(
1068                f"The scores produced by the evaluator must match the scores expected by the eval. Got: [{', '.join(self.scores.keys())}] and expected: [{', '.join(output_score_keys)}]"
1069            )
1070
1071        problems = validate_scores_against_output_scores(
1072            self.scores, eval.output_scores
1073        )
1074        if problems:
1075            raise ValueError(problems[0])
1076        return self
1077
1078    @model_validator(mode="after")
1079    def validate_reference_answer(self) -> Self:
1080        parent_eval_config = self.parent_eval_config()
1081        if parent_eval_config and parent_eval_config.config_type == EvalConfigType.v2:
1082            return self
1083        parent_eval = parent_eval_config.parent_eval() if parent_eval_config else None
1084        if not parent_eval:
1085            return self
1086
1087        evaluation_data_type = parent_eval.evaluation_data_type
1088        if (
1089            self.reference_answer is not None
1090            and evaluation_data_type is not None
1091            and evaluation_data_type != EvalDataType.reference_answer
1092        ):
1093            raise ValueError(
1094                f"reference_answer is only valid for reference answer evals. Got: {evaluation_data_type.value}"
1095            )
1096        return self
1097
1098
1099class EvalConfig(KilnParentedModel, KilnParentModel, parent_of={"runs": EvalRun}):
1100    """
1101    A configuration for running an eval. This includes anything needed to run the eval on a dataset like the prompt, model, thresholds, etc.
1102
1103    A eval might have many configs, example running the same eval with 2 different models. Comparing eval results is only valid within the scope of the same config.
1104    """
1105
1106    name: FilenameString = Field(description="The name of the eval config.")
1107    model_name: str | None = Field(
1108        default=None,
1109        description="The name of the model to use for this eval config. Required for legacy configs, None for V2.",
1110    )
1111    model_provider: str | None = Field(
1112        default=None,
1113        description="The provider of the model to use for this eval config. Required for legacy configs, None for V2.",
1114    )
1115    config_type: EvalConfigType = Field(
1116        default=EvalConfigType.g_eval,
1117        description="This is used to determine the type of eval to run.",
1118    )
1119    properties: dict[str, Any] | V2EvalConfigProperties | None = Field(
1120        default=None,
1121        description="Properties to be used to execute the eval config. Legacy configs use a dict; V2 configs use typed properties.",
1122    )
1123
1124    @model_validator(mode="before")
1125    @classmethod
1126    def dispatch_properties_parsing(cls, data: Any, info: ValidationInfo) -> Any:
1127        # The union lists dict first, so a raw dict always stays a plain dict —
1128        # even one whose keys happen to match a typed V2 shape (legacy configs
1129        # store arbitrary dicts). V2 configs persist properties as a dict too,
1130        # so parse those into the typed union here, before field validation.
1131        if not isinstance(data, dict):
1132            return data
1133        if data.get("config_type", EvalConfigType.g_eval) == EvalConfigType.v2:
1134            # code_eval stores its score() source in a sibling scorer.py: on file
1135            # load, the type-gated helper parses those props through
1136            # CodeEvalProperties (reading the sibling via the load context) so a
1137            # bad or missing scorer.py surfaces directly instead of being masked
1138            # by a union fallback. Other property types pass through unchanged.
1139            data = _eager_parse_code_eval_on_load(data, info.context or {})
1140            props = data.get("properties")
1141            if isinstance(props, dict):
1142                data = dict(data)
1143                data["properties"] = _V2_PROPERTIES_ADAPTER.validate_python(props)
1144        return data
1145
1146    def parent_eval(self) -> Union["Eval", None]:
1147        if self.parent is not None and self.parent.__class__.__name__ != "Eval":
1148            raise ValueError("parent must be an Eval")
1149        return self.parent  # type: ignore
1150
1151    def runs(self, readonly: bool = False) -> list[EvalRun]:
1152        return super().runs(readonly=readonly)  # type: ignore
1153
1154    @model_validator(mode="after")
1155    def validate_properties(self) -> Self:
1156        if self.config_type in (EvalConfigType.g_eval, EvalConfigType.llm_as_judge):
1157            if not isinstance(self.properties, dict):
1158                raise ValueError("Legacy config properties must be a dict")
1159            if "eval_steps" not in self.properties or not isinstance(
1160                self.properties["eval_steps"], list
1161            ):
1162                raise ValueError("eval_steps is required and must be a list for g_eval")
1163            if "task_description" in self.properties and not isinstance(
1164                self.properties["task_description"], str
1165            ):
1166                raise ValueError(
1167                    "task_description is optional, but if provided must be a string"
1168                )
1169            if self.model_name is None or self.model_provider is None:
1170                raise ValueError(
1171                    "model_name and model_provider are required for legacy configs"
1172                )
1173            return self
1174        elif self.config_type == EvalConfigType.v2:
1175            if not isinstance(self.properties, BaseModel):
1176                raise ValueError("V2 config requires typed properties")
1177            if self.model_name is not None or self.model_provider is not None:
1178                raise ValueError(
1179                    "V2 configs must not set root-level model_name/model_provider"
1180                )
1181            return self
1182        else:
1183            raise ValueError(f"Invalid eval config type: {self.config_type}")
1184
1185    @model_validator(mode="after")
1186    def validate_v2_templates_and_expressions(self) -> Self:
1187        if self.config_type != EvalConfigType.v2 or not isinstance(
1188            self.properties, BaseModel
1189        ):
1190            return self
1191
1192        from kiln_ai.utils.jinja_engine import (
1193            compile_expression_or_raise,
1194            compile_template_or_raise,
1195        )
1196
1197        props = self.properties
1198        if isinstance(props, LlmJudgeProperties):
1199            compile_template_or_raise(props.prompt_template)
1200            from jinja2 import meta
1201
1202            from kiln_ai.utils.jinja_engine import _template_env
1203
1204            referenced = meta.find_undeclared_variables(
1205                _template_env.parse(props.prompt_template)
1206            )
1207            meaningful = {"final_message", "trace", "task_input"}
1208            if not (referenced & meaningful):
1209                raise ValueError(
1210                    "prompt_template never references the model output. "
1211                    "A template that uses only reference_data (or no variables) "
1212                    "produces the same judge prompt for every run. "
1213                    "Reference the output, e.g. {{ final_message }}."
1214                )
1215
1216        if isinstance(
1217            props,
1218            (
1219                ExactMatchProperties,
1220                PatternMatchProperties,
1221                ContainsProperties,
1222                SetCheckProperties,
1223            ),
1224        ):
1225            if props.value_expression is not None:
1226                compile_expression_or_raise(props.value_expression)
1227
1228        return self
1229
1230    @model_validator(mode="after")
1231    def validate_json_serializable(self) -> "EvalConfig":
1232        if self.config_type == EvalConfigType.v2:
1233            return self
1234        if self.properties is None:
1235            return self
1236        try:
1237            json.dumps(self.properties, ensure_ascii=False)
1238        except TypeError as e:
1239            raise ValueError(f"Properties must be JSON serializable: {e!s}")
1240        return self
1241
1242
1243class EvalDataType(str, Enum):
1244    """The type of task output data to evaluate."""
1245
1246    final_answer = "final_answer"
1247    full_trace = "full_trace"
1248    reference_answer = "reference_answer"
1249
1250
1251class TaskRunSplit(BaseModel):
1252    """A split whose items are TaskRuns, selected by a dataset filter."""
1253
1254    # Fields a future build adds are preserved rather than dropped, for the same reason
1255    # Eval.splits keeps unknown split names: these files sync between app versions. It is
1256    # also why the legacy-field migration never overwrites a split that `splits` already
1257    # describes — rebuilding one from a bare filter-id string would drop everything else
1258    # on it.
1259    model_config = ConfigDict(extra="allow")
1260
1261    source: Literal["task_run"] = "task_run"
1262    filter_id: DatasetFilterId
1263
1264
1265class EvalInputSplit(BaseModel):
1266    """A split whose items are EvalInputs, selected by an eval-input filter."""
1267
1268    model_config = ConfigDict(extra="allow")
1269
1270    source: Literal["eval_input"] = "eval_input"
1271    filter_id: EvalInputFilterId
1272
1273
1274SplitRef = Annotated[
1275    Union[TaskRunSplit, EvalInputSplit],
1276    Discriminator("source"),
1277]
1278"""One of an eval's splits: which store its items come from, and which filter selects them.
1279Discriminated on `source`, so a split's backing is part of its value rather than a
1280convention a reader has to know."""
1281
1282EvalSplitName = Literal["train", "val", "test"]
1283"""The split names the API exposes. `Eval.splits` is keyed by plain `str` so a file
1284written by a build that knows a fourth split still loads here (see Eval.splits)."""
1285
1286LEGACY_SPLIT_FIELDS: Dict[str, str] = {
1287    "test": "eval_set_filter_id",
1288    "train": "train_set_filter_id",
1289}
1290"""Split name -> the deprecated flat `Eval` field a Kiln build predating `splits` stored it
1291in. These fields are an input format and nothing else: `Eval.migrate_legacy_split_fields`
1292reads each one once, on the way in, and clears it. Nothing else in the codebase reads or
1293writes them, and they are never written to disk again — see `Eval.splits`."""
1294
1295
1296class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}):
1297    """An evaluator definition that specifies what to evaluate and how scores should be produced."""
1298
1299    name: FilenameString = Field(description="The name of the eval.")
1300    description: str | None = Field(
1301        default=None, description="The description of the eval"
1302    )
1303    template: EvalTemplateId | None = Field(
1304        default=None,
1305        description="The template selected when creating this eval. Useful for suggesting eval steps and output scores.",
1306    )
1307    current_config_id: ID_TYPE = Field(
1308        default=None,
1309        description="The id of the current config to use for this eval. This can be changed over time to run the same eval with different configs.",
1310    )
1311    eval_set_filter_id: DatasetFilterId | None = Field(
1312        default=None,
1313        deprecated=True,
1314        description="Deprecated, and neither read nor written. It exists only so evals written by a Kiln build that predates `splits` still load: on load its value is migrated into splits['test'] once, and the field is then cleared. It is always saved as null. Read splits['test'] instead.",
1315    )
1316    eval_configs_filter_id: DatasetFilterId | None = Field(
1317        default=None,
1318        description="The id of the dataset filter which defines which dataset items are included when comparing the quality of the eval configs under this eval. Should consist of dataset items with ratings.",
1319    )
1320    train_set_filter_id: DatasetFilterId | None = Field(
1321        default=None,
1322        deprecated=True,
1323        description="Deprecated, and neither read nor written. It exists only so evals written by a Kiln build that predates `splits` still load: on load its value is migrated into splits['train'] once, and the field is then cleared. It is always saved as null. Read splits['train'] instead.",
1324    )
1325    splits: Dict[str, SplitRef] = Field(
1326        default_factory=dict,
1327        description="The eval's dataset splits, keyed by split name ('test', 'train', 'val'), and the only place they are stored. Each split names the store its items come from and the filter that selects them. Keys this build doesn't know are preserved but not exposed. 'golden' is not a split and does not belong here: the golden set must be dataset (TaskRun) based, because human ratings only exist on dataset items, so it is stored in eval_configs_filter_id instead. Nothing reads splits['golden'] — writing it is accepted and silently ignored. In Python, prefer Eval.set_split() to assigning into this dict: it refuses to mutate a readonly (cached) eval, and marks the field as set so exclude_unset dumps keep it.",
1328    )
1329    output_scores: List[EvalOutputScore] = Field(
1330        description="The scores this evaluator should produce."
1331    )
1332    favourite: bool = Field(
1333        default=False,
1334        description="Whether this eval is a favourite of the user. Rendered as a star icon in the UI.",
1335    )
1336    priority: Priority | None = Field(
1337        default=None,
1338        description="The priority of the eval. None on evals created before priority lived on evals; read through resolved_priority(), which falls back to the associated spec.",
1339    )
1340    status: EvalStatus | None = Field(
1341        default=None,
1342        description="The status of the eval. None on evals created before status lived on evals; read through resolved_status(), which falls back to the associated spec.",
1343    )
1344    template_properties: dict[str, str | int | bool | float] | None = Field(
1345        default=None,
1346        description="Properties to be used to execute the eval. This is template_type specific and should serialize to a json dict.",
1347    )
1348    evaluation_data_type: EvalDataType | None = Field(
1349        default=EvalDataType.final_answer,
1350        description="The output of the task run to evaluate. Can be final answer, full trace, or None for V2 evals.",
1351    )
1352
1353    @model_validator(mode="after")
1354    def migrate_legacy_split_fields(self) -> Self:
1355        """Migrate the deprecated flat filter fields into `splits`, once, and clear them.
1356
1357        `splits` is the only home a split has. These fields are an input format for evals
1358        written before it existed, so each one is read exactly once — here — and only for
1359        a split `splits` doesn't already describe. `splits` winning is what makes the
1360        migration one-way: once a value is in `splits` it is the eval's answer, and a
1361        legacy field left over beside it (a hand-edited file, or one an older build wrote
1362        after a newer one) is ignored rather than allowed to overwrite it. Overwriting
1363        would also drop any extra fields on the existing split object, which
1364        `TaskRunSplit`/`EvalInputSplit` keep on purpose (`extra="allow"`).
1365
1366        Both fields are then cleared, unconditionally. That is what makes this a
1367        migration rather than a second home: nothing downstream can read a stale value,
1368        the eval saves with both fields null, and re-running the validator — which
1369        `validate_assignment` does on every attribute set, including `self.path = path`
1370        at the end of save_to_file — has nothing left to do. An older Kiln build reading
1371        the saved file sees no test set rather than the wrong one; that is the accepted
1372        cost of a single home, and the eval list surfaces the evals it can't read.
1373
1374        Reads and writes go through `__dict__` because the fields are
1375        `deprecated=True`: attribute access on them emits a DeprecationWarning, which is
1376        meant for callers, not for the one place that is supposed to touch them.
1377
1378        Must stay declared before validate_splits, which requires a test split: an eval
1379        that carries only legacy fields gets its test split from here.
1380        """
1381        for name, field_name in LEGACY_SPLIT_FIELDS.items():
1382            filter_id = self.__dict__.get(field_name)
1383            if filter_id is not None and name not in self.splits:
1384                self.splits[name] = TaskRunSplit(filter_id=filter_id)
1385                # The split now lives only in `splits`, so an exclude_unset dump has to
1386                # carry it: on a legacy eval `splits` was never explicitly set.
1387                self.__pydantic_fields_set__.add("splits")
1388            self.__dict__[field_name] = None
1389        return self
1390
1391    @model_validator(mode="after")
1392    def validate_splits(self) -> Self:
1393        if "test" not in self.splits:
1394            raise ValueError("An eval must have a test split. Set splits['test'].")
1395        return self
1396
1397    def set_split(self, name: str, split: SplitRef) -> None:
1398        """Set one of the eval's splits.
1399
1400        Equivalent to `eval.splits[name] = split` plus the two things item assignment on
1401        a dict can't do for itself, because it never reaches `__setattr__`: refusing to
1402        mutate a readonly (cached) eval, and marking `splits` as set so an
1403        exclude_unset dump still carries it.
1404        """
1405        # Readonly instances are the cached ones, shared with every other holder of the
1406        # same file, so this check has to be explicit here.
1407        self._ensure_not_readonly("splits")
1408        self.splits[name] = split
1409        # Validated evals always have `splits` marked already (their test split came from
1410        # `splits` or from the legacy migration, which marks it), so this is for instances
1411        # built by model_construct, where nothing did.
1412        self.__pydantic_fields_set__.add("splits")
1413
1414    # Workaround to return typed parent without importing Task
1415    def parent_task(self) -> Union["Task", None]:
1416        if self.parent is not None and self.parent.__class__.__name__ != "Task":
1417            raise ValueError("parent must be a Task")
1418        return self.parent  # type: ignore
1419
1420    def configs(self, readonly: bool = False) -> list[EvalConfig]:
1421        return super().configs(readonly=readonly)  # type: ignore
1422
1423    # Workaround to return typed parent without importing Spec
1424    def associated_spec(self, readonly: bool = False) -> Union["Spec", None]:
1425        """
1426        Get the spec associated with this eval, if any.
1427        Returns None for legacy evals that are not associated with a spec.
1428        """
1429
1430        task = self.parent_task()
1431        if not task or not self.id:
1432            return None
1433
1434        specs = task.specs(readonly=readonly)
1435        for spec in specs:
1436            if spec.eval_id == self.id:
1437                return spec
1438        return None
1439
1440    def resolved_priority(self, spec: Union["Spec", None] = None) -> Priority:
1441        """
1442        The eval's effective priority. Priority lives on the eval; evals created
1443        before that (spec-backed legacy files) fall back to their spec's value.
1444        Pass *spec* when the caller already has it, to avoid a re-scan.
1445        """
1446        if self.priority is not None:
1447            return self.priority
1448        spec = spec or self.associated_spec(readonly=True)
1449        if spec is not None:
1450            return spec.priority
1451        return Priority.p1
1452
1453    def resolved_status(self, spec: Union["Spec", None] = None) -> EvalStatus:
1454        """
1455        The eval's effective status, with the same spec fallthrough as
1456        resolved_priority().
1457        """
1458        if self.status is not None:
1459            return self.status
1460        spec = spec or self.associated_spec(readonly=True)
1461        if spec is not None:
1462            return spec.status
1463        return EvalStatus.active
1464
1465    def eval_reference_data_keys(self) -> list[str]:
1466        """Union of reference-data keys across all of this eval's V2 configs.
1467
1468        Returns deduplicated keys in stable insertion order.
1469        """
1470        seen: set[str] = set()
1471        result: list[str] = []
1472        for config in self.configs(readonly=True):
1473            if config.config_type != EvalConfigType.v2:
1474                continue
1475            if not isinstance(config.properties, V2_PROPERTY_TYPES):
1476                continue
1477            for key in reference_data_keys(config.properties):  # type: ignore[arg-type]
1478                if key not in seen:
1479                    seen.add(key)
1480                    result.append(key)
1481        return result
1482
1483    @model_validator(mode="after")
1484    def upgrade_old_reference_answer_eval_config(self) -> Self:
1485        """
1486        Migration: Set the first judge config as the default for existing reference answer evals that don't have a current_config_id set.
1487
1488        For reference_answer evals that don't have a current_config_id set, this migration
1489        will set the first config (by created_at) as the default.
1490        """
1491        if self.id is None:
1492            return self
1493
1494        # Only run during file loading
1495        if not self._loaded_from_file:
1496            return self
1497
1498        # Skip if already migrated (has a current_config_id set)
1499        if self.current_config_id is not None:
1500            return self
1501
1502        # Only migrate reference_answer evals
1503        if self.evaluation_data_type != EvalDataType.reference_answer:
1504            return self
1505
1506        # Prevent recursion: self.configs() loads child files, which re-loads this parent
1507        # (see basemodel.py where we iterate_children_paths_of_parent_path calls load_from_file)
1508        # This causes the validator to run again, creating an infinite loop without this guard.
1509        with _migration_lock:
1510            if self.id in _currently_migrating_eval_ids:
1511                return self
1512            _currently_migrating_eval_ids.add(self.id)
1513
1514        try:
1515            # Get the configs - these are loaded from child files
1516            configs_list = self.configs(readonly=True)
1517            if configs_list and len(configs_list) > 0:
1518                # Sort by created_at to get the oldest (first created) config
1519                sorted_configs = sorted(configs_list, key=lambda c: c.created_at)
1520                self.current_config_id = sorted_configs[0].id
1521        finally:
1522            with _migration_lock:
1523                _currently_migrating_eval_ids.discard(self.id)
1524
1525        return self
1526
1527    @model_validator(mode="after")
1528    def validate_scores(self) -> Self:
1529        if self.output_scores is None or len(self.output_scores) == 0:
1530            raise ValueError(
1531                "output_scores are required, and must have at least one score."
1532            )
1533
1534        # check for duplicate names (once transformed to JSON keys)
1535        output_score_keys = [score.json_key() for score in self.output_scores]
1536        if len(output_score_keys) != len(set(output_score_keys)):
1537            raise ValueError(
1538                f"output_scores must have unique names (once transformed to JSON keys). Got: [{', '.join(output_score_keys)}]"
1539            )
1540        return self
1541
1542    @model_validator(mode="after")
1543    def validate_template_properties(self) -> Self:
1544        if self.template is None:
1545            return self
1546
1547        if (
1548            self.template is not EvalTemplateId.rag
1549            and self.eval_configs_filter_id is None
1550        ):
1551            raise ValueError(
1552                "eval_configs_filter_id is required for all templates except 'rag'"
1553            )
1554
1555        # For spec-based evals, template_properties will be None and validation happens in the spec
1556        # For legacy evals, template_properties contains the data and we validate here
1557        if self.template_properties is None:
1558            return self
1559
1560        # Check for properties that are required for the issue template (legacy evals only)
1561        if self.template == EvalTemplateId.issue:
1562            if "issue_prompt" not in self.template_properties or not isinstance(
1563                self.template_properties["issue_prompt"], str
1564            ):
1565                raise ValueError("issue_prompt is required for issue template")
1566            if "failure_example" in self.template_properties and not isinstance(
1567                self.template_properties["failure_example"], str
1568            ):
1569                raise ValueError(
1570                    "failure_example is optional for issue template, but if provided must be a string"
1571                )
1572            if "pass_example" in self.template_properties and not isinstance(
1573                self.template_properties["pass_example"], str
1574            ):
1575                raise ValueError(
1576                    "pass_example is optional for issue template, but if provided must be a string"
1577                )
1578
1579        if self.template == EvalTemplateId.tool_call:
1580            if self.evaluation_data_type != EvalDataType.full_trace:
1581                raise ValueError(
1582                    "tool_call template should have evaluation_data_type set to full_trace"
1583                )
1584            if (
1585                "tool" not in self.template_properties
1586                or not isinstance(self.template_properties["tool"], str)
1587                or not self.template_properties["tool"].strip()
1588            ):
1589                raise ValueError("tool is required for tool call template")
1590            if "tool_function_name" not in self.template_properties or not isinstance(
1591                self.template_properties["tool_function_name"], str
1592            ):
1593                raise ValueError(
1594                    "tool_function_name is required for tool call template"
1595                )
1596            if (
1597                "appropriate_tool_use_guidelines" not in self.template_properties
1598                or not isinstance(
1599                    self.template_properties["appropriate_tool_use_guidelines"], str
1600                )
1601                or not self.template_properties[
1602                    "appropriate_tool_use_guidelines"
1603                ].strip()
1604            ):
1605                raise ValueError(
1606                    "appropriate_tool_use_guidelines is required for tool call template"
1607                )
1608            if (
1609                "inappropriate_tool_use_guidelines" in self.template_properties
1610                and not isinstance(
1611                    self.template_properties["inappropriate_tool_use_guidelines"], str
1612                )
1613            ):
1614                raise ValueError(
1615                    "inappropriate_tool_use_guidelines is optional for tool call template, but if provided must be a string"
1616                )
1617        return self