kiln_ai.datamodel.eval

   1import json
   2from enum import Enum
   3from threading import Lock
   4from typing import TYPE_CHECKING, Annotated, Any, Dict, List, Literal, Union
   5
   6from pydantic import (
   7    BaseModel,
   8    ConfigDict,
   9    Discriminator,
  10    Field,
  11    GetJsonSchemaHandler,
  12    JsonValue,
  13    SerializationInfo,
  14    SerializerFunctionWrapHandler,
  15    ValidationInfo,
  16    model_serializer,
  17    model_validator,
  18)
  19from pydantic.json_schema import JsonSchemaValue
  20from pydantic_core import CoreSchema
  21from typing_extensions import Self
  22
  23from kiln_ai.datamodel.basemodel import (
  24    ID_TYPE,
  25    FilenameString,
  26    FilenameStringShort,
  27    KilnParentedModel,
  28    KilnParentModel,
  29)
  30from kiln_ai.datamodel.code_file_storage import (
  31    read_code_from_sibling_file,
  32    write_code_to_sibling_file,
  33)
  34from kiln_ai.datamodel.datamodel_enums import (
  35    EvalStatus,
  36    Priority,
  37    TaskOutputRatingType,
  38)
  39from kiln_ai.datamodel.dataset_filters import DatasetFilterId, EvalInputFilterId
  40from kiln_ai.datamodel.json_schema import string_to_json_key
  41from kiln_ai.datamodel.task_run import Usage
  42from kiln_ai.datamodel.tool_id import ToolId, validate_tool_allowlist
  43from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
  44
  45if TYPE_CHECKING:
  46    from kiln_ai.datamodel.spec import Spec
  47    from kiln_ai.datamodel.task import Task
  48    from kiln_ai.datamodel.task_run import TaskRun
  49
  50EvalScores = Dict[str, float]
  51
  52# Module-level set to track evals currently being migrated (to prevent recursion)
  53# Protected by _migration_lock to ensure thread-safe access
  54_migration_lock = Lock()
  55_currently_migrating_eval_ids: set[ID_TYPE] = set()
  56
  57# Fixed name of the sibling file that holds a code judge's Python source, stored
  58# beside its eval_config.kiln. Fixed so authored tests can `from scorer import score`.
  59SCORER_CODE_FILENAME = "scorer.py"
  60
  61
  62class EvalTemplateId(str, Enum):
  63    """
  64    An eval template is a pre-defined eval that can be used as a starting point for a new eval.
  65    """
  66
  67    kiln_requirements = "kiln_requirements"
  68    desired_behaviour = "desired_behaviour"
  69    issue = "kiln_issue"
  70    tool_call = "tool_call"
  71    toxicity = "toxicity"
  72    bias = "bias"
  73    maliciousness = "maliciousness"
  74    factual_correctness = "factual_correctness"
  75    jailbreak = "jailbreak"
  76    rag = "rag"
  77
  78
  79class EvalConfigType(str, Enum):
  80    """The type of eval configuration, determining how scores are generated."""
  81
  82    g_eval = "g_eval"
  83    llm_as_judge = "llm_as_judge"
  84    v2 = "v2"
  85
  86
  87class V2EvalType(str, Enum):
  88    """V2-only eval type enum. Each value maps to a typed properties class
  89    and a V2 adapter."""
  90
  91    llm_judge = "llm_judge"
  92    exact_match = "exact_match"
  93    pattern_match = "pattern_match"
  94    set_check = "set_check"
  95    tool_call_check = "tool_call_check"
  96    contains = "contains"
  97    step_count_check = "step_count_check"
  98    code_eval = "code_eval"
  99
 100
 101class LlmJudgeProperties(BaseModel):
 102    type: Literal[V2EvalType.llm_judge] = V2EvalType.llm_judge
 103    model_name: str
 104    model_provider: str
 105    system_prompt: str | None = None
 106    prompt_template: str
 107    reference_keys: list[str] = []
 108    thinking_instruction: str | None = None
 109    g_eval: bool = False
 110    # User-written evaluation steps, bound to {{ judge_instructions }} when the
 111    # prompt template is rendered. Used by evals with no spec or template to
 112    # derive default steps from.
 113    judge_instructions: list[str] | None = None
 114
 115
 116class ExactMatchProperties(BaseModel):
 117    type: Literal[V2EvalType.exact_match] = V2EvalType.exact_match
 118    value_expression: str | None = None
 119    expected_value: str | None = None
 120    reference_key: str | None = Field(default=None, min_length=1)
 121    case_sensitive: bool = True
 122
 123    @model_validator(mode="after")
 124    def validate_value_source(self) -> Self:
 125        if (self.expected_value is None) == (self.reference_key is None):
 126            raise ValueError(
 127                "Exactly one of expected_value or reference_key must be set"
 128            )
 129        return self
 130
 131
 132class PatternMatchProperties(BaseModel):
 133    type: Literal[V2EvalType.pattern_match] = V2EvalType.pattern_match
 134    value_expression: str | None = None
 135    pattern: str
 136    mode: Literal["must_match", "must_not_match"] = "must_match"
 137
 138    @model_validator(mode="after")
 139    def validate_pattern(self) -> Self:
 140        import re
 141
 142        try:
 143            re.compile(self.pattern)
 144        except re.error as e:
 145            raise ValueError(f"Invalid regex pattern '{self.pattern}': {e}") from e
 146        return self
 147
 148
 149class ContainsProperties(BaseModel):
 150    type: Literal[V2EvalType.contains] = V2EvalType.contains
 151    value_expression: str | None = None
 152    substring: str | None = None
 153    reference_key: str | None = Field(default=None, min_length=1)
 154    case_sensitive: bool = True
 155    mode: Literal["must_contain", "must_not_contain"] = "must_contain"
 156
 157    @model_validator(mode="after")
 158    def validate_value_source(self) -> Self:
 159        if (self.substring is None) == (self.reference_key is None):
 160            raise ValueError("Exactly one of substring or reference_key must be set")
 161        return self
 162
 163
 164class SetCheckProperties(BaseModel):
 165    type: Literal[V2EvalType.set_check] = V2EvalType.set_check
 166    value_expression: str | None = None
 167    expected_set: list[str] | None = None
 168    reference_key: str | None = Field(default=None, min_length=1)
 169    mode: Literal["subset", "superset", "equal"]
 170
 171    @model_validator(mode="after")
 172    def validate_value_source(self) -> Self:
 173        if (self.expected_set is None) == (self.reference_key is None):
 174            raise ValueError("Exactly one of expected_set or reference_key must be set")
 175        return self
 176
 177
 178class ArgMatch(BaseModel):
 179    value: JsonValue
 180    match_mode: Literal["exact", "contains", "regex"] = "exact"
 181
 182    @model_validator(mode="after")
 183    def validate_regex(self) -> Self:
 184        if self.match_mode == "regex":
 185            import re
 186
 187            try:
 188                re.compile(str(self.value))
 189            except re.error as e:
 190                raise ValueError(f"Invalid regex value '{self.value}': {e}") from e
 191        return self
 192
 193
 194class ToolCallSpec(BaseModel):
 195    tool_name: str
 196    expected_args: dict[str, ArgMatch] | None = None
 197
 198
 199class ToolCallCheckProperties(BaseModel):
 200    type: Literal[V2EvalType.tool_call_check] = V2EvalType.tool_call_check
 201    expected_tools: list[ToolCallSpec] = Field(min_length=1)
 202    match_mode: Literal["any", "all", "ordered", "never"] = "all"
 203    on_unexpected_tools: Literal["ignore", "fail"] = "ignore"
 204
 205
 206class StepCountCheckProperties(BaseModel):
 207    type: Literal[V2EvalType.step_count_check] = V2EvalType.step_count_check
 208    count_type: Literal["tool_calls", "model_responses", "turns"]
 209    min_count: int | None = None
 210    max_count: int | None = None
 211
 212    @model_validator(mode="after")
 213    def check_bounds(self) -> Self:
 214        if self.min_count is None and self.max_count is None:
 215            raise ValueError(
 216                "step_count_check requires at least one of min_count / max_count"
 217            )
 218        if (
 219            self.min_count is not None
 220            and self.max_count is not None
 221            and self.min_count > self.max_count
 222        ):
 223            raise ValueError("min_count must be <= max_count")
 224        return self
 225
 226
 227class CodeEvalProperties(BaseModel):
 228    type: Literal[V2EvalType.code_eval] = V2EvalType.code_eval
 229    code: str
 230    reference_keys: list[str] = []
 231    timeout_seconds: int = Field(default=180, ge=1, le=300)
 232    tool_allowlist: list[ToolId] = Field(
 233        default_factory=list,
 234        description="Explicit per-tool allowlist of tools the scorer code may call.",
 235    )
 236
 237    @model_validator(mode="after")
 238    def validate_allowlist(self) -> Self:
 239        # No self-reference check: a code eval is not itself a tool.
 240        validate_tool_allowlist(self.tool_allowlist, caller="code evals")
 241        return self
 242
 243    @model_validator(mode="before")
 244    @classmethod
 245    def _read_code_file(cls, data: Any, info: ValidationInfo) -> Any:
 246        """When loading from disk, inject `code` from the sibling scorer.py.
 247
 248        The source is stored in scorer.py beside eval_config.kiln, not inline in
 249        the JSON. CodeEvalProperties is a nested member of the
 250        V2EvalConfigProperties discriminated union in EvalConfig.properties, so
 251        the load context set on the parent EvalConfig (`source_dir`) propagates
 252        down to this validator. The shared helper reads the file here, before
 253        field validation, so the existing validate_code trio runs against the
 254        loaded string unchanged.
 255        """
 256        # Explicit type-gate (defense-in-depth): this validator only ever runs
 257        # for code_eval properties — it lives on CodeEvalProperties, and both the
 258        # discriminated union and the eager parse route only code_eval dicts
 259        # here. Assert that gate so a future refactor can't quietly read
 260        # scorer.py for another eval type. None (type omitted, field defaults)
 261        # and the enum form both pass; only a present, mismatched type is
 262        # rejected, so valid-input behavior is unchanged.
 263        if isinstance(data, dict) and data.get("type") not in (
 264            None,
 265            V2EvalType.code_eval.value,
 266        ):
 267            raise ValueError(
 268                "CodeEvalProperties can only load code_eval properties, "
 269                f"got type: {data.get('type')!r}"
 270            )
 271        return read_code_from_sibling_file(
 272            data,
 273            info.context or {},
 274            filename=SCORER_CODE_FILENAME,
 275            kiln_filename="eval_config.kiln",
 276            model_label="CodeEvalProperties",
 277        )
 278
 279    @model_serializer(mode="wrap")
 280    def _serialize(
 281        self, handler: SerializerFunctionWrapHandler, info: SerializationInfo
 282    ) -> dict[str, Any]:
 283        """On disk-save, write `code` to scorer.py and omit it from the .kiln JSON.
 284
 285        Delegates to the shared sibling-file helper, which uses the same save
 286        context attachments use (`save_attachments` + `dest_path`); it propagates
 287        from the parent EvalConfig's save_to_file() down to this nested union
 288        member. Without that context — normal model_dump / API responses —
 289        `code` is left in the output and no file is written, so the API contract
 290        is unchanged. The default handler preserves `type` (needed by the
 291        discriminator), `reference_keys`, and `timeout_seconds`.
 292
 293        Schema note: a custom model_serializer would otherwise collapse the
 294        *serialization-mode* JSON schema to an untyped object
 295        (`model_json_schema(mode="serialization")` loses per-field typing).
 296        Unlike CodeTool — which is never a FastAPI response_model — this model is
 297        nested in EvalConfig, and EvalConfig IS the declared `response_model` on
 298        several endpoints (eval_api.py). FastAPI generates response schemas in
 299        serialization mode, so a collapsed schema here would split
 300        CodeEvalProperties into an untyped `-Output` component and drift the
 301        checked-in api_schema.d.ts (breaking check_schema.sh and the web types
 302        that key off `components["schemas"]["CodeEvalProperties"]`). The
 303        `__get_pydantic_json_schema__` override below is therefore REQUIRED (not
 304        optional): it keeps the serialization-mode schema identical to
 305        validation mode. Do not remove either the serializer (runtime file
 306        storage) or the override (schema stability).
 307        """
 308        return write_code_to_sibling_file(
 309            handler(self),
 310            info.context or {},
 311            filename=SCORER_CODE_FILENAME,
 312            code=self.code,
 313        )
 314
 315    @classmethod
 316    def __get_pydantic_json_schema__(
 317        cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler
 318    ) -> JsonSchemaValue:
 319        """Keep the serialization-mode JSON schema identical to validation mode.
 320
 321        The wrap serializer above returns an untyped `dict`, which would collapse
 322        this model's serialization-mode JSON schema to `{additionalProperties:
 323        true, type: object}` (dropping `code`, `type`, etc.). Because EvalConfig
 324        (which nests this model) is a FastAPI response_model, that collapse would
 325        drift the committed OpenAPI/api_schema.d.ts. Dropping the `serialization`
 326        core-schema entries makes JSON-schema generation use the field-based
 327        (validation) representation in both modes, so `code` stays present and
 328        typed and there is no `-Input`/`-Output` split. The custom serializer
 329        lives on the inner `model` core schema (the before/after validators wrap
 330        it in function schemas), so the strip must be recursive. This affects
 331        only schema generation, never runtime (de)serialization.
 332        """
 333
 334        def strip_serialization(schema: Any) -> Any:
 335            if isinstance(schema, dict):
 336                return {
 337                    key: strip_serialization(value)
 338                    for key, value in schema.items()
 339                    if key != "serialization"
 340                }
 341            if isinstance(schema, list):
 342                return [strip_serialization(item) for item in schema]
 343            return schema
 344
 345        return handler(strip_serialization(core_schema))
 346
 347    @model_validator(mode="after")
 348    def validate_code(self) -> Self:
 349        code_bytes = self.code.encode("utf-8")
 350        if len(code_bytes) > 64 * 1024:
 351            raise ValueError(
 352                f"Code is too large ({len(code_bytes)} bytes). Maximum size is 64KB."
 353            )
 354
 355        try:
 356            compile(self.code, "<code_eval>", "exec")
 357        except SyntaxError as e:
 358            raise ValueError(f"Code has a syntax error: {e}") from e
 359
 360        import ast
 361
 362        tree = ast.parse(self.code)
 363        # Both sync and async score functions are accepted here.
 364        # Async coroutines are transparently awaited in sandbox_worker.execute_scorer_bridged.
 365        has_score_fn = any(
 366            isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
 367            and node.name == "score"
 368            for node in ast.iter_child_nodes(tree)
 369        )
 370        if not has_score_fn:
 371            raise ValueError(
 372                "Code must define a module-level 'score' function (def score(...))."
 373            )
 374
 375        return self
 376
 377
 378V2EvalConfigProperties = Annotated[
 379    Union[
 380        LlmJudgeProperties,
 381        ExactMatchProperties,
 382        PatternMatchProperties,
 383        SetCheckProperties,
 384        ToolCallCheckProperties,
 385        ContainsProperties,
 386        StepCountCheckProperties,
 387        CodeEvalProperties,
 388    ],
 389    Discriminator("type"),
 390]
 391
 392# Explicit tuple of V2 property types for isinstance() checks.
 393# Must list exactly the same types as the V2EvalConfigProperties union above.
 394V2_PROPERTY_TYPES: tuple[type[BaseModel], ...] = (
 395    LlmJudgeProperties,
 396    ExactMatchProperties,
 397    PatternMatchProperties,
 398    SetCheckProperties,
 399    ToolCallCheckProperties,
 400    ContainsProperties,
 401    StepCountCheckProperties,
 402    CodeEvalProperties,
 403)
 404
 405
 406def reference_data_keys(props: V2EvalConfigProperties) -> list[str]:
 407    """Return the reference-data keys a single judge needs.
 408
 409    Exhaustive match over the V2 properties union: adding a new V2 type
 410    without handling it here will fail ``ty`` type-checking.
 411    """
 412    match props:
 413        case ExactMatchProperties():
 414            return [props.reference_key] if props.reference_key else []
 415        case ContainsProperties():
 416            return [props.reference_key] if props.reference_key else []
 417        case SetCheckProperties():
 418            return [props.reference_key] if props.reference_key else []
 419        case LlmJudgeProperties():
 420            return list(props.reference_keys)
 421        case CodeEvalProperties():
 422            return list(props.reference_keys)
 423        case PatternMatchProperties():
 424            return []
 425        case ToolCallCheckProperties():
 426            return []
 427        case StepCountCheckProperties():
 428            return []
 429        case _:
 430            raise_exhaustive_enum_error(props)
 431
 432
 433def _eager_parse_code_eval_on_load(
 434    data: dict[str, Any], ctx: dict[str, Any]
 435) -> dict[str, Any]:
 436    """Eagerly parse a code_eval EvalConfig's `properties` on file load.
 437
 438    V2 code judges store their score() source in a sibling scorer.py, not inline
 439    in the JSON. On load, parse a code_eval properties dict through
 440    CodeEvalProperties (which reads scorer.py via the propagated load context) so
 441    any error surfaces directly. Without this, the outer
 442    `V2EvalConfigProperties | dict | None` union would recover from the nested
 443    member's error by falling back to the dict branch, masking the real cause
 444    (e.g. a missing scorer.py or a bad score() function) behind a generic
 445    "V2 config requires typed properties". See functional spec §2.2 / §4.
 446
 447    Only touches code_eval properties during a file load, gated explicitly on
 448    `type == code_eval`; every other input passes through unchanged. Lifted
 449    verbatim from EvalConfig.dispatch_properties_parsing so the code-eval load
 450    path is a clearly-named, code-eval-local step rather than smeared into the
 451    generic dispatcher.
 452    """
 453    if not ctx.get("loading_from_file"):
 454        return data
 455    props = data.get("properties")
 456    if isinstance(props, dict) and props.get("type") == V2EvalType.code_eval.value:
 457        data = dict(data)
 458        data["properties"] = CodeEvalProperties.model_validate(props, context=ctx)
 459    return data
 460
 461
 462def validate_scores_against_output_scores(
 463    scores: EvalScores,
 464    output_scores: list["EvalOutputScore"],
 465) -> list[str]:
 466    """Validate that *scores* fall within the expected range for each output score.
 467
 468    Returns a list of human-readable problem strings (empty list means all OK).
 469    This is a pure function — it does NOT raise; callers decide how to surface errors.
 470    """
 471
 472    def _is_numeric(v: object) -> bool:
 473        return isinstance(v, (int, float)) and not isinstance(v, bool)
 474
 475    problems: list[str] = []
 476    for output_score in output_scores:
 477        key = output_score.json_key()
 478        if key not in scores:
 479            continue
 480        value = scores[key]
 481
 482        match output_score.type:
 483            case TaskOutputRatingType.five_star:
 484                if not _is_numeric(value) or value < 1.0 or value > 5.0:
 485                    problems.append(
 486                        f"Score {output_score.name} is a five_star rating and must be a number between 1.0 and 5.0 inclusive. Got: {value}"
 487                    )
 488            case TaskOutputRatingType.pass_fail:
 489                if not _is_numeric(value) or value < 0.0 or value > 1.0:
 490                    problems.append(
 491                        f"Score {output_score.name} is a pass_fail rating and must be a number between 0.0 and 1.0 inclusive. Got: {value}"
 492                    )
 493            case TaskOutputRatingType.pass_fail_critical:
 494                if not _is_numeric(value) or value < -1.0 or value > 1.0:
 495                    problems.append(
 496                        f"Score {output_score.name} is a pass_fail_critical rating and must be a number between -1.0 and 1.0 inclusive. Got: {value}"
 497                    )
 498            case TaskOutputRatingType.custom:
 499                problems.append(
 500                    f"Custom scores are not supported in evaluators. '{output_score.name}' was set to a custom score."
 501                )
 502            case _:
 503                raise_exhaustive_enum_error(output_score.type)
 504    return problems
 505
 506
 507class SkippedReason(str, Enum):
 508    """Terminal skip reasons stored as str for back/forward-compat."""
 509
 510    missing_reference_key = "missing_reference_key"
 511    extraction_failed = "extraction_failed"
 512    missing_trace = "missing_trace"
 513    incompatible_input_shape = "incompatible_input_shape"
 514    code_eval_not_trusted = "code_eval_not_trusted"
 515    type_not_available = "type_not_available"
 516
 517
 518class V2EvalResult(BaseModel):
 519    """Result of a single V2 eval ``evaluate()`` call."""
 520
 521    scores: EvalScores = Field(default_factory=dict)
 522    skipped_reason: SkippedReason | None = None
 523    skipped_detail: str | None = None
 524    intermediate_outputs: Dict[str, str] | None = None
 525    usage: Usage | None = Field(
 526        default=None,
 527        description="What the judgment itself cost, if it called a model. None for the deterministic eval types, which call none. Stored on the resulting EvalRun as eval_usage.",
 528    )
 529
 530
 531class UserMessage(BaseModel):
 532    text: str
 533
 534
 535class SingleTurnEvalInputData(BaseModel):
 536    type: Literal["single_turn"] = "single_turn"
 537    user_message: UserMessage
 538
 539
 540class MultiTurnSyntheticEvalInputData(BaseModel):
 541    type: Literal["multi_turn_synthetic"] = "multi_turn_synthetic"
 542    first_message: UserMessage | None = None
 543    synthetic_user_info: dict[str, JsonValue] = {}
 544
 545
 546EvalInputData = Annotated[
 547    Union[
 548        SingleTurnEvalInputData,
 549        MultiTurnSyntheticEvalInputData,
 550    ],
 551    Discriminator("type"),
 552]
 553
 554
 555class EvalInput(KilnParentedModel):
 556    """A single evaluation input item, stored as a child of a Task.
 557
 558    Each EvalInput contains the data needed to run an evaluation (e.g. a user
 559    message) plus optional reference data for comparison and tags for filtering.
 560    """
 561
 562    data: EvalInputData = Field(
 563        description="The input data for this eval item.",
 564    )
 565    reference: dict[str, JsonValue] | None = Field(
 566        default=None,
 567        description="Optional reference data (ground truth) for this eval input, keyed by reference name.",
 568    )
 569    tags: list[str] = Field(
 570        default_factory=list,
 571        description="Tags for filtering eval inputs.",
 572    )
 573
 574
 575class EvalTaskInput(BaseModel):
 576    """The runtime data bundle passed to V2 evaluators.
 577
 578    Assembled by the eval runner from the item being evaluated and the task run that
 579    was scored. The item is either an EvalInput or a TaskRun drawn from the dataset;
 580    which one it is determines where `reference_data` and `task_input` come from.
 581    """
 582
 583    final_message: str = Field(
 584        description="The final model output (task output text).",
 585    )
 586    trace: list[dict[str, Any]] | None = Field(
 587        default=None,
 588        description="The full conversation trace, if available.",
 589    )
 590    reference_data: dict[str, JsonValue] | None = Field(
 591        default=None,
 592        description=(
 593            "Ground-truth data for the item being evaluated, keyed by reference name. "
 594            "Taken from EvalInput.reference for an EvalInput-backed item; for a "
 595            "TaskRun-backed dataset item it is the item's own stored output under the "
 596            "key 'reference_answer', since that output is the curated answer. None "
 597            "when a TaskRun is scored as itself (judge calibration), where the item "
 598            "and the scored run are the same record."
 599        ),
 600    )
 601    task_input: str | None = Field(
 602        default=None,
 603        description="The original task input text.",
 604    )
 605
 606    @classmethod
 607    def from_trace(
 608        cls, trace: "TaskRun", source: "TaskRun | EvalInput"
 609    ) -> "EvalTaskInput":
 610        """What a judge sees: the trace that was produced, plus the item it came from.
 611
 612        The two are separate arguments because they are separate records once eval traces
 613        live on their own TaskRun — the trace holds what the model said, and the source
 614        item holds the ground truth to compare it against. They are the same object only
 615        for calibration, where the golden dataset item is itself what gets scored.
 616        """
 617        from kiln_ai.datamodel.task_run import TaskRun as _TaskRun
 618
 619        if not isinstance(trace, _TaskRun):
 620            raise TypeError("Expected a TaskRun instance for trace")
 621
 622        trace_data: list[dict[str, Any]] | None = None
 623        if trace.trace is not None:
 624            trace_data = [dict(msg) for msg in trace.trace]
 625
 626        if isinstance(source, EvalInput):
 627            if not isinstance(source.data, SingleTurnEvalInputData):
 628                raise ValueError("EvalTaskInput only supports single-turn EvalInput")
 629            reference_data = source.reference
 630            # The item's own text, not the trace's: an EvalInput is the canonical
 631            # statement of the input, and the adapter may have reserialized it.
 632            task_input = source.data.user_message.text
 633        elif isinstance(source, _TaskRun):
 634            # A TaskRun-backed dataset item stores the curated answer as its output, so
 635            # that output is the ground truth to compare the trace against. Skipped when
 636            # source *is* trace (calibration, and `from_task_run`): there the golden item
 637            # is itself what gets scored, so a reference would be byte-identical to
 638            # `final_message` and every judge comparing them would pass.
 639            reference_data = (
 640                None if source is trace else {"reference_answer": source.output.output}
 641            )
 642            task_input = trace.input
 643        else:
 644            raise TypeError("Expected a TaskRun or EvalInput instance for source")
 645
 646        return cls(
 647            final_message=trace.output.output,
 648            trace=trace_data,
 649            reference_data=reference_data,
 650            task_input=task_input,
 651        )
 652
 653    @classmethod
 654    def from_task_run(cls, task_run: "TaskRun") -> "EvalTaskInput":
 655        """A TaskRun scored as itself, with no separate source item."""
 656        return cls.from_trace(task_run, task_run)
 657
 658    @classmethod
 659    def from_eval_input(
 660        cls, eval_input: "EvalInput", run_output: "TaskRun"
 661    ) -> "EvalTaskInput":
 662        """A generated run scored against the EvalInput it was generated from."""
 663        if not isinstance(eval_input, EvalInput):
 664            raise TypeError("Expected an EvalInput instance")
 665        return cls.from_trace(run_output, eval_input)
 666
 667
 668class EvalOutputScore(BaseModel):
 669    """
 670    A definition of a score that an evaluator will produce.
 671
 672    Very similar to TaskRequirement, but conceptually different keeping in a separate models.
 673    """
 674
 675    name: FilenameStringShort = Field(
 676        description="The name of the score. Will be provided to the model so use a descriptive name. Should align to the model's TaskRequirement name if you want to use human evals to evaluate the evaluator's performance."
 677    )
 678    instruction: str | None = Field(
 679        default=None,
 680        description="A description of the score, used to help the model understand the goal of the score. Will be provided to evaluator models, so should be written for the model, not the team/user.",
 681    )
 682    type: TaskOutputRatingType = Field(
 683        description="The type of rating to use ('five_star', 'pass_fail', 'pass_fail_critical').",
 684    )
 685
 686    def json_key(self) -> str:
 687        """
 688        The JSON key for the score, used when running the evaluator with a LLM and we need JSON output.
 689
 690        For example, "Overall Rating" -> "overall_rating"
 691        """
 692        return string_to_json_key(self.name)
 693
 694    @model_validator(mode="after")
 695    def validate_type(self) -> Self:
 696        if self.type == TaskOutputRatingType.custom:
 697            raise ValueError(
 698                f"Custom scores are not supported in evaluators. Score '{self.name}' was set to a custom score."
 699            )
 700        return self
 701
 702
 703LEGACY_TRACE_FIELDS = ("output", "task_run_trace", "task_run_usage", "reference_answer")
 704"""The EvalRun fields that hold a copy of the trace a score was computed over.
 705
 706Deprecated: new records point at a TaskRun with `scored_run_id` instead. Kept declared
 707and loadable forever - every record written before the split still carries them. A
 708pointer-mode record must leave all of them None, which `validate_record_mode` enforces.
 709`input` is deprecated alongside these but is not in this tuple: it is the one a legacy
 710record is *required* to have, so it is checked separately.
 711
 712Three ways to mark a pydantic field deprecated; these fields use two of them:
 713
 7141. A `DEPRECATED:` prefix in the `description`. Reaches a human reading the SDK docs or
 715   the OpenAPI schema, and nothing else. Used.
 7162. `json_schema_extra={"deprecated": True}`. Puts `"deprecated": true` in the JSON
 717   schema, which `openapi-typescript` turns into a `/** @deprecated */` JSDoc tag, so
 718   the TS compiler and editors strike through every web call site. No runtime effect.
 719   Used.
 7203. `Field(deprecated=True)`. Same schema output as (2), but pydantic also raises a
 721   DeprecationWarning on every attribute *read* — and reading these is the correct,
 722   permanent way to render a legacy record, so it would be a warning storm. Not used.
 723   Do not "fix" this to (3) without silencing that first; (2) already provides the
 724   tooling signal (3) would be reached for."""
 725
 726
 727class EvalRun(KilnParentedModel):
 728    """
 729    The scores an eval produced for a single dataset item.
 730
 731    This is a child of an EvalConfig, which specifies how the scores were generated.
 732
 733    Eval runs can be one of 2 types:
 734    1) eval_config_eval=False (scoring): we were evaluating a task run config (a method of running the task). We take the item's input, run the task with the task_run_config, then run the evaluator on that output. task_run_config_id must be set.
 735    2) eval_config_eval=True (calibration): we were evaluating an eval config (a method of evaluating the task). We used an existing human-rated dataset item's input/output, and ran the evaluator on it. task_run_config_id must be None.
 736
 737    A record is described by two independent facts — whether it points at a TaskRun, and
 738    whether it was skipped — which `validate_record_mode` constrains to three legal
 739    shapes. What is exclusive is where the trace lives: on the record, or on the TaskRun,
 740    never both.
 741
 742    - **Pointer** (new): `scored_run_id` names the TaskRun that holds the trace. All
 743      inline trace fields must be None.
 744    - **Skipped**: `skipped_reason` set, so scores are not required. It also carries a
 745      `scored_run_id` if the trace existed and only scoring was skipped — so a skip can
 746      be a pointer record too — and none if the skip happened before generation.
 747    - **Legacy inline**: no `scored_run_id`; the trace lives on this record, and `input`
 748      is required unless the record was skipped. Every record written before the
 749      trace/score split is in this state, and it stays valid forever.
 750    """
 751
 752    dataset_id: ID_TYPE | None = Field(
 753        default=None,
 754        description="The ID of the dataset item (TaskRun) that was used for this run. Mutually exclusive with eval_input_id.",
 755    )
 756    scored_run_id: ID_TYPE | None = Field(
 757        default=None,
 758        description="The ID of the TaskRun this score was computed over. None for legacy records that carry their trace inline. A dangling reference is tolerated: the score still renders and still aggregates, only the trace drill-through is unavailable.",
 759    )
 760    task_run_config_id: ID_TYPE | None = Field(
 761        description="The ID of the TaskRunConfig that was run, if this eval run was based on a task run. Must belong to the same Task as this eval. Can be None if this eval run is based on an eval config."
 762    )
 763    eval_config_eval: bool = Field(
 764        description="Whether this eval run to evaluate the parent eval config (evaluating the config using an existing dataset item). If true, task_run_config_id must be None, as we're not running the task.",
 765        default=False,
 766    )
 767    input: str | None = Field(
 768        default=None,
 769        json_schema_extra={"deprecated": True},
 770        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.input instead. The input to the task. JSON formatted for structured input, plaintext for unstructured input. Required on legacy records (those with neither a scored_run_id nor a skipped_reason), never set on new ones.",
 771    )
 772    output: str | None = Field(
 773        default=None,
 774        json_schema_extra={"deprecated": True},
 775        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.output.output instead. The output of the task. None for skipped-before-execution runs.",
 776    )
 777    reference_answer: str | None = Field(
 778        default=None,
 779        json_schema_extra={"deprecated": True},
 780        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id. The reference answer for the input. JSON formatted for structured reference answer, plaintext for unstructured reference answer. Used for reference answer evals.",
 781    )
 782    intermediate_outputs: Dict[str, str] | None = Field(
 783        default=None,
 784        description="The intermediate outputs of the task (example, eval thinking).",
 785    )
 786    task_run_trace: str | None = Field(
 787        default=None,
 788        json_schema_extra={"deprecated": True},
 789        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.trace instead. The JSON formatted trace of the task run that produced the output.",
 790    )
 791    scores: EvalScores = Field(
 792        default={},
 793        description="The output scores of the evaluator (aligning to those required by the grand-parent Eval this object is a child of).",
 794    )
 795    task_run_usage: Usage | None = Field(
 796        default=None,
 797        json_schema_extra={"deprecated": True},
 798        description="DEPRECATED: the trace now lives on the TaskRun named by scored_run_id; read TaskRun.usage instead. The usage of the task run that produced this eval run output (not the usage by the evaluation model).",
 799    )
 800    eval_usage: Usage | None = Field(
 801        default=None,
 802        description="The usage of the evaluation model (judge) that produced this eval run's scores, aggregated across every LLM call the judgment made. Distinct from task_run_usage, which is the evaluated task run's usage. None for non-LLM evals (e.g. code evals) and for records that predate this field.",
 803    )
 804
 805    eval_input_id: ID_TYPE | None = Field(
 806        default=None,
 807        description="ID of the EvalInput used for this run (V2 evals). Mutually exclusive with dataset_id.",
 808    )
 809    skipped_reason: str | None = Field(
 810        default=None,
 811        description="If set, this run was skipped. Stored as str for back/forward-compat; conventionally a SkippedReason value.",
 812    )
 813    skipped_detail: str | None = Field(
 814        default=None,
 815        description="Case-specific detail for skipped runs (e.g. missing key name).",
 816    )
 817
 818    def parent_eval_config(self) -> Union["EvalConfig", None]:
 819        if self.parent is not None and self.parent.__class__.__name__ != "EvalConfig":
 820            raise ValueError("parent must be an EvalConfig")
 821        return self.parent  # type: ignore
 822
 823    @model_validator(mode="after")
 824    def validate_input_source(self) -> Self:
 825        if (self.dataset_id is None) == (self.eval_input_id is None):
 826            raise ValueError(
 827                "Exactly one of dataset_id (V1 TaskRun source) or "
 828                "eval_input_id (V2 EvalInput source) must be set"
 829            )
 830        return self
 831
 832    @model_validator(mode="after")
 833    def validate_record_mode(self) -> Self:
 834        """Keep the three record states (pointer / skipped / legacy inline) exclusive.
 835
 836        The forbidding half of the pointer rule is the one that earns its keep: a record
 837        that points at a TaskRun must never also carry a second copy of what it scored,
 838        which nothing would keep in sync.
 839        """
 840        inline_set = [f for f in LEGACY_TRACE_FIELDS if getattr(self, f) is not None]
 841
 842        if self.scored_run_id is not None:
 843            # Checked before the skip branch on purpose: a record skipped at *scoring*
 844            # time still has a scored_run_id, and still must not carry inline data.
 845            if self.input is not None or inline_set:
 846                carried = (["input"] if self.input is not None else []) + inline_set
 847                raise ValueError(
 848                    "An EvalRun with scored_run_id must not carry inline trace data "
 849                    f"(set: {', '.join(carried)}). "
 850                    "The trace lives on the referenced TaskRun."
 851                )
 852            return self
 853
 854        if self.skipped_reason is not None:
 855            # Skipped before generation: there is nothing to point at, and nothing to
 856            # require. Legacy skipped records that do carry inline data stay valid.
 857            return self
 858
 859        if self.input is None:
 860            raise ValueError("A legacy EvalRun (no scored_run_id) requires input.")
 861        return self
 862
 863    @model_validator(mode="after")
 864    def validate_output_fields(self) -> Self:
 865        # Resolved before the pointer bypass below, so the pointer path can't skip the
 866        # parent-type check.
 867        parent_eval_config = self.parent_eval_config()
 868        if self.scored_run_id is not None:
 869            # Pointer mode: the output lives on the referenced TaskRun, and
 870            # validate_record_mode has already required it to be absent here.
 871            return self
 872        if parent_eval_config and parent_eval_config.config_type == EvalConfigType.v2:
 873            return self
 874        parent_eval = parent_eval_config.parent_eval() if parent_eval_config else None
 875        if not parent_eval:
 876            return self
 877
 878        if self.output is None and self.skipped_reason is None:
 879            raise ValueError("V1 EvalRun requires output to be set")
 880
 881        evaluation_data_type = parent_eval.evaluation_data_type
 882        if (
 883            evaluation_data_type == EvalDataType.final_answer
 884            and self.task_run_trace is not None
 885        ):
 886            raise ValueError("final_answer runs should not set trace")
 887        elif (
 888            not self.eval_config_eval
 889            and evaluation_data_type == EvalDataType.full_trace
 890            and self.task_run_trace is None
 891        ):
 892            raise ValueError("full_trace task run eval runs should include trace")
 893
 894        return self
 895
 896    @model_validator(mode="after")
 897    def validate_eval_run_types(self) -> Self:
 898        if self.eval_config_eval and self.task_run_config_id is not None:
 899            raise ValueError(
 900                "task_run_config_id must be None if eval_config_eval is true"
 901            )
 902        if not self.eval_config_eval and self.task_run_config_id is None:
 903            raise ValueError(
 904                "task_run_config_id must be set if eval_config_eval is false"
 905            )
 906        return self
 907
 908    @model_validator(mode="after")
 909    def validate_scores(self) -> Self:
 910        if self.skipped_reason is not None:
 911            return self
 912
 913        if self.scores is None or len(self.scores) == 0:
 914            raise ValueError("scores are required, and must have at least one score.")
 915
 916        parent_eval_config = self.parent_eval_config()
 917        eval = parent_eval_config.parent_eval() if parent_eval_config else None
 918        if not eval:
 919            return self
 920
 921        output_score_keys = [score.json_key() for score in eval.output_scores]
 922        if set(output_score_keys) != set(self.scores.keys()):
 923            raise ValueError(
 924                f"The scores produced by the evaluator must match the scores expected by the eval. Got: [{', '.join(self.scores.keys())}] and expected: [{', '.join(output_score_keys)}]"
 925            )
 926
 927        problems = validate_scores_against_output_scores(
 928            self.scores, eval.output_scores
 929        )
 930        if problems:
 931            raise ValueError(problems[0])
 932        return self
 933
 934    @model_validator(mode="after")
 935    def validate_reference_answer(self) -> Self:
 936        parent_eval_config = self.parent_eval_config()
 937        if parent_eval_config and parent_eval_config.config_type == EvalConfigType.v2:
 938            return self
 939        parent_eval = parent_eval_config.parent_eval() if parent_eval_config else None
 940        if not parent_eval:
 941            return self
 942
 943        evaluation_data_type = parent_eval.evaluation_data_type
 944        if (
 945            self.reference_answer is not None
 946            and evaluation_data_type is not None
 947            and evaluation_data_type != EvalDataType.reference_answer
 948        ):
 949            raise ValueError(
 950                f"reference_answer is only valid for reference answer evals. Got: {evaluation_data_type.value}"
 951            )
 952        return self
 953
 954
 955class EvalConfig(KilnParentedModel, KilnParentModel, parent_of={"runs": EvalRun}):
 956    """
 957    A configuration for running an eval. This includes anything needed to run the eval on a dataset like the prompt, model, thresholds, etc.
 958
 959    A eval might have many configs, example running the same eval with 2 different models. Comparing eval results is only valid within the scope of the same config.
 960    """
 961
 962    name: FilenameString = Field(description="The name of the eval config.")
 963    model_name: str | None = Field(
 964        default=None,
 965        description="The name of the model to use for this eval config. Required for legacy configs, None for V2.",
 966    )
 967    model_provider: str | None = Field(
 968        default=None,
 969        description="The provider of the model to use for this eval config. Required for legacy configs, None for V2.",
 970    )
 971    config_type: EvalConfigType = Field(
 972        default=EvalConfigType.g_eval,
 973        description="This is used to determine the type of eval to run.",
 974    )
 975    properties: V2EvalConfigProperties | dict[str, Any] | None = Field(
 976        default=None,
 977        description="Properties to be used to execute the eval config. Legacy configs use a dict; V2 configs use typed properties.",
 978    )
 979
 980    @model_validator(mode="before")
 981    @classmethod
 982    def dispatch_properties_parsing(cls, data: Any, info: ValidationInfo) -> Any:
 983        # Pydantic's discriminated-union parsing would reject a plain dict for
 984        # `properties` because dicts don't carry a discriminator field. V1 (legacy)
 985        # configs store properties as an untyped dict, so we shallow-copy and
 986        # re-assign it here to force Pydantic to accept the dict branch of the union.
 987        if not isinstance(data, dict):
 988            return data
 989        config_type = data.get("config_type", "g_eval")
 990        if config_type != "v2":
 991            props = data.get("properties")
 992            if props is not None and isinstance(props, dict):
 993                data = dict(data)
 994                data["properties"] = props
 995            return data
 996
 997        # V2: the only load-time special-case is code_eval, whose score() source
 998        # lives in a sibling scorer.py. Delegate to the code-eval-local helper,
 999        # which is explicitly type-gated (`type == code_eval`); all other V2
1000        # properties pass through unchanged.
1001        return _eager_parse_code_eval_on_load(data, info.context or {})
1002
1003    def parent_eval(self) -> Union["Eval", None]:
1004        if self.parent is not None and self.parent.__class__.__name__ != "Eval":
1005            raise ValueError("parent must be an Eval")
1006        return self.parent  # type: ignore
1007
1008    def runs(self, readonly: bool = False) -> list[EvalRun]:
1009        return super().runs(readonly=readonly)  # type: ignore
1010
1011    @model_validator(mode="after")
1012    def validate_properties(self) -> Self:
1013        if self.config_type in (EvalConfigType.g_eval, EvalConfigType.llm_as_judge):
1014            if not isinstance(self.properties, dict):
1015                raise ValueError("Legacy config properties must be a dict")
1016            if "eval_steps" not in self.properties or not isinstance(
1017                self.properties["eval_steps"], list
1018            ):
1019                raise ValueError("eval_steps is required and must be a list for g_eval")
1020            if "task_description" in self.properties and not isinstance(
1021                self.properties["task_description"], str
1022            ):
1023                raise ValueError(
1024                    "task_description is optional, but if provided must be a string"
1025                )
1026            if self.model_name is None or self.model_provider is None:
1027                raise ValueError(
1028                    "model_name and model_provider are required for legacy configs"
1029                )
1030            return self
1031        elif self.config_type == EvalConfigType.v2:
1032            if not isinstance(self.properties, BaseModel):
1033                raise ValueError("V2 config requires typed properties")
1034            if self.model_name is not None or self.model_provider is not None:
1035                raise ValueError(
1036                    "V2 configs must not set root-level model_name/model_provider"
1037                )
1038            return self
1039        else:
1040            raise ValueError(f"Invalid eval config type: {self.config_type}")
1041
1042    @model_validator(mode="after")
1043    def validate_v2_templates_and_expressions(self) -> Self:
1044        if self.config_type != EvalConfigType.v2 or not isinstance(
1045            self.properties, BaseModel
1046        ):
1047            return self
1048
1049        from kiln_ai.utils.jinja_engine import (
1050            compile_expression_or_raise,
1051            compile_template_or_raise,
1052        )
1053
1054        props = self.properties
1055        if isinstance(props, LlmJudgeProperties):
1056            compile_template_or_raise(props.prompt_template)
1057            from jinja2 import meta
1058
1059            from kiln_ai.utils.jinja_engine import _template_env
1060
1061            referenced = meta.find_undeclared_variables(
1062                _template_env.parse(props.prompt_template)
1063            )
1064            meaningful = {"final_message", "trace", "task_input"}
1065            if not (referenced & meaningful):
1066                raise ValueError(
1067                    "prompt_template never references the model output. "
1068                    "A template that uses only reference_data (or no variables) "
1069                    "produces the same judge prompt for every run. "
1070                    "Reference the output, e.g. {{ final_message }}."
1071                )
1072
1073        if isinstance(
1074            props,
1075            (
1076                ExactMatchProperties,
1077                PatternMatchProperties,
1078                ContainsProperties,
1079                SetCheckProperties,
1080            ),
1081        ):
1082            if props.value_expression is not None:
1083                compile_expression_or_raise(props.value_expression)
1084
1085        return self
1086
1087    @model_validator(mode="after")
1088    def validate_json_serializable(self) -> "EvalConfig":
1089        if self.config_type == EvalConfigType.v2:
1090            return self
1091        if self.properties is None:
1092            return self
1093        try:
1094            json.dumps(self.properties, ensure_ascii=False)
1095        except TypeError as e:
1096            raise ValueError(f"Properties must be JSON serializable: {e!s}")
1097        return self
1098
1099
1100class EvalDataType(str, Enum):
1101    """The type of task output data to evaluate."""
1102
1103    final_answer = "final_answer"
1104    full_trace = "full_trace"
1105    reference_answer = "reference_answer"
1106
1107
1108class TaskRunSplit(BaseModel):
1109    """A split whose items are TaskRuns, selected by a dataset filter."""
1110
1111    # Fields a future build adds are preserved rather than dropped, for the same reason
1112    # Eval.splits keeps unknown split names: these files sync between app versions. It is
1113    # also why the legacy-field migration never overwrites a split that `splits` already
1114    # describes — rebuilding one from a bare filter-id string would drop everything else
1115    # on it.
1116    model_config = ConfigDict(extra="allow")
1117
1118    source: Literal["task_run"] = "task_run"
1119    filter_id: DatasetFilterId
1120
1121
1122class EvalInputSplit(BaseModel):
1123    """A split whose items are EvalInputs, selected by an eval-input filter."""
1124
1125    model_config = ConfigDict(extra="allow")
1126
1127    source: Literal["eval_input"] = "eval_input"
1128    filter_id: EvalInputFilterId
1129
1130
1131SplitRef = Annotated[
1132    Union[TaskRunSplit, EvalInputSplit],
1133    Discriminator("source"),
1134]
1135"""One of an eval's splits: which store its items come from, and which filter selects them.
1136Discriminated on `source`, so a split's backing is part of its value rather than a
1137convention a reader has to know."""
1138
1139EvalSplitName = Literal["train", "val", "test"]
1140"""The split names the API exposes. `Eval.splits` is keyed by plain `str` so a file
1141written by a build that knows a fourth split still loads here (see Eval.splits)."""
1142
1143LEGACY_SPLIT_FIELDS: Dict[str, str] = {
1144    "test": "eval_set_filter_id",
1145    "train": "train_set_filter_id",
1146}
1147"""Split name -> the deprecated flat `Eval` field a Kiln build predating `splits` stored it
1148in. These fields are an input format and nothing else: `Eval.migrate_legacy_split_fields`
1149reads each one once, on the way in, and clears it. Nothing else in the codebase reads or
1150writes them, and they are never written to disk again — see `Eval.splits`."""
1151
1152
1153class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}):
1154    """An evaluator definition that specifies what to evaluate and how scores should be produced."""
1155
1156    name: FilenameString = Field(description="The name of the eval.")
1157    description: str | None = Field(
1158        default=None, description="The description of the eval"
1159    )
1160    template: EvalTemplateId | None = Field(
1161        default=None,
1162        description="The template selected when creating this eval. Useful for suggesting eval steps and output scores.",
1163    )
1164    current_config_id: ID_TYPE = Field(
1165        default=None,
1166        description="The id of the current config to use for this eval. This can be changed over time to run the same eval with different configs.",
1167    )
1168    eval_set_filter_id: DatasetFilterId | None = Field(
1169        default=None,
1170        deprecated=True,
1171        description="Deprecated, and neither read nor written. It exists only so evals written by a Kiln build that predates `splits` still load: on load its value is migrated into splits['test'] once, and the field is then cleared. It is always saved as null. Read splits['test'] instead.",
1172    )
1173    eval_configs_filter_id: DatasetFilterId | None = Field(
1174        default=None,
1175        description="The id of the dataset filter which defines which dataset items are included when comparing the quality of the eval configs under this eval. Should consist of dataset items with ratings.",
1176    )
1177    train_set_filter_id: DatasetFilterId | None = Field(
1178        default=None,
1179        deprecated=True,
1180        description="Deprecated, and neither read nor written. It exists only so evals written by a Kiln build that predates `splits` still load: on load its value is migrated into splits['train'] once, and the field is then cleared. It is always saved as null. Read splits['train'] instead.",
1181    )
1182    splits: Dict[str, SplitRef] = Field(
1183        default_factory=dict,
1184        description="The eval's dataset splits, keyed by split name ('test', 'train', 'val'), and the only place they are stored. Each split names the store its items come from and the filter that selects them. Keys this build doesn't know are preserved but not exposed. 'golden' is not a split and does not belong here: the golden set must be dataset (TaskRun) based, because human ratings only exist on dataset items, so it is stored in eval_configs_filter_id instead. Nothing reads splits['golden'] — writing it is accepted and silently ignored. In Python, prefer Eval.set_split() to assigning into this dict: it refuses to mutate a readonly (cached) eval, and marks the field as set so exclude_unset dumps keep it.",
1185    )
1186    output_scores: List[EvalOutputScore] = Field(
1187        description="The scores this evaluator should produce."
1188    )
1189    favourite: bool = Field(
1190        default=False,
1191        description="Whether this eval is a favourite of the user. Rendered as a star icon in the UI.",
1192    )
1193    priority: Priority | None = Field(
1194        default=None,
1195        description="The priority of the eval. None on evals created before priority lived on evals; read through resolved_priority(), which falls back to the associated spec.",
1196    )
1197    status: EvalStatus | None = Field(
1198        default=None,
1199        description="The status of the eval. None on evals created before status lived on evals; read through resolved_status(), which falls back to the associated spec.",
1200    )
1201    template_properties: dict[str, str | int | bool | float] | None = Field(
1202        default=None,
1203        description="Properties to be used to execute the eval. This is template_type specific and should serialize to a json dict.",
1204    )
1205    evaluation_data_type: EvalDataType | None = Field(
1206        default=EvalDataType.final_answer,
1207        description="The output of the task run to evaluate. Can be final answer, full trace, or None for V2 evals.",
1208    )
1209
1210    @model_validator(mode="before")
1211    @classmethod
1212    def migrate_eval_input_filter_id(cls, data: Any) -> Any:
1213        """Migrate the pre-`splits` `eval_input_filter_id` key into an EvalInput-backed test split.
1214
1215        A third legacy input for the test split, so it follows the same rule as the two
1216        declared legacy fields: it fills the test split only when `splits` does not
1217        already describe one, and is dropped either way (it is not a declared field, so
1218        it is never written back).
1219
1220        FUTURE: Safe to delete whenever someone wants to. Only internal projects contained
1221        this key and none of them still exist; no public project file has ever had it, so
1222        this never becomes a compatibility commitment.
1223        """
1224        if not isinstance(data, dict):
1225            return data
1226        filter_id = data.get("eval_input_filter_id")
1227        if filter_id is None:
1228            return data
1229        if data.get("eval_set_filter_id") is not None:
1230            # Two legacy inputs naming one split with two different backings. `splits`
1231            # winning resolves legacy-vs-`splits` disagreements, but not this one: both
1232            # sides here are legacy, so there is no rule that picks between them, and
1233            # silently dropping either is worse than refusing the file.
1234            raise ValueError(
1235                "An eval cannot set both eval_set_filter_id and eval_input_filter_id: they are two backings for the same test split."
1236            )
1237        data = dict(data)
1238        data.pop("eval_input_filter_id")
1239        splits = dict(data.get("splits") or {})
1240        if "test" not in splits:
1241            splits["test"] = {"source": "eval_input", "filter_id": filter_id}
1242        data["splits"] = splits
1243        return data
1244
1245    @model_validator(mode="after")
1246    def migrate_legacy_split_fields(self) -> Self:
1247        """Migrate the deprecated flat filter fields into `splits`, once, and clear them.
1248
1249        `splits` is the only home a split has. These fields are an input format for evals
1250        written before it existed, so each one is read exactly once — here — and only for
1251        a split `splits` doesn't already describe. `splits` winning is what makes the
1252        migration one-way: once a value is in `splits` it is the eval's answer, and a
1253        legacy field left over beside it (a hand-edited file, or one an older build wrote
1254        after a newer one) is ignored rather than allowed to overwrite it. Overwriting
1255        would also drop any extra fields on the existing split object, which
1256        `TaskRunSplit`/`EvalInputSplit` keep on purpose (`extra="allow"`).
1257
1258        Both fields are then cleared, unconditionally. That is what makes this a
1259        migration rather than a second home: nothing downstream can read a stale value,
1260        the eval saves with both fields null, and re-running the validator — which
1261        `validate_assignment` does on every attribute set, including `self.path = path`
1262        at the end of save_to_file — has nothing left to do. An older Kiln build reading
1263        the saved file sees no test set rather than the wrong one; that is the accepted
1264        cost of a single home, and the eval list surfaces the evals it can't read.
1265
1266        Reads and writes go through `__dict__` because the fields are
1267        `deprecated=True`: attribute access on them emits a DeprecationWarning, which is
1268        meant for callers, not for the one place that is supposed to touch them.
1269
1270        Must stay declared before validate_splits, which requires a test split: an eval
1271        that carries only legacy fields gets its test split from here.
1272        """
1273        for name, field_name in LEGACY_SPLIT_FIELDS.items():
1274            filter_id = self.__dict__.get(field_name)
1275            if filter_id is not None and name not in self.splits:
1276                self.splits[name] = TaskRunSplit(filter_id=filter_id)
1277                # The split now lives only in `splits`, so an exclude_unset dump has to
1278                # carry it: on a legacy eval `splits` was never explicitly set.
1279                self.__pydantic_fields_set__.add("splits")
1280            self.__dict__[field_name] = None
1281        return self
1282
1283    @model_validator(mode="after")
1284    def validate_splits(self) -> Self:
1285        if "test" not in self.splits:
1286            raise ValueError("An eval must have a test split. Set splits['test'].")
1287        return self
1288
1289    def set_split(self, name: str, split: SplitRef) -> None:
1290        """Set one of the eval's splits.
1291
1292        Equivalent to `eval.splits[name] = split` plus the two things item assignment on
1293        a dict can't do for itself, because it never reaches `__setattr__`: refusing to
1294        mutate a readonly (cached) eval, and marking `splits` as set so an
1295        exclude_unset dump still carries it.
1296        """
1297        # Readonly instances are the cached ones, shared with every other holder of the
1298        # same file, so this check has to be explicit here.
1299        self._ensure_not_readonly("splits")
1300        self.splits[name] = split
1301        # Validated evals always have `splits` marked already (their test split came from
1302        # `splits` or from the legacy migration, which marks it), so this is for instances
1303        # built by model_construct, where nothing did.
1304        self.__pydantic_fields_set__.add("splits")
1305
1306    # Workaround to return typed parent without importing Task
1307    def parent_task(self) -> Union["Task", None]:
1308        if self.parent is not None and self.parent.__class__.__name__ != "Task":
1309            raise ValueError("parent must be a Task")
1310        return self.parent  # type: ignore
1311
1312    def configs(self, readonly: bool = False) -> list[EvalConfig]:
1313        return super().configs(readonly=readonly)  # type: ignore
1314
1315    # Workaround to return typed parent without importing Spec
1316    def associated_spec(self, readonly: bool = False) -> Union["Spec", None]:
1317        """
1318        Get the spec associated with this eval, if any.
1319        Returns None for legacy evals that are not associated with a spec.
1320        """
1321
1322        task = self.parent_task()
1323        if not task or not self.id:
1324            return None
1325
1326        specs = task.specs(readonly=readonly)
1327        for spec in specs:
1328            if spec.eval_id == self.id:
1329                return spec
1330        return None
1331
1332    def resolved_priority(self, spec: Union["Spec", None] = None) -> Priority:
1333        """
1334        The eval's effective priority. Priority lives on the eval; evals created
1335        before that (spec-backed legacy files) fall back to their spec's value.
1336        Pass *spec* when the caller already has it, to avoid a re-scan.
1337        """
1338        if self.priority is not None:
1339            return self.priority
1340        spec = spec or self.associated_spec(readonly=True)
1341        if spec is not None:
1342            return spec.priority
1343        return Priority.p1
1344
1345    def resolved_status(self, spec: Union["Spec", None] = None) -> EvalStatus:
1346        """
1347        The eval's effective status, with the same spec fallthrough as
1348        resolved_priority().
1349        """
1350        if self.status is not None:
1351            return self.status
1352        spec = spec or self.associated_spec(readonly=True)
1353        if spec is not None:
1354            return spec.status
1355        return EvalStatus.active
1356
1357    def eval_reference_data_keys(self) -> list[str]:
1358        """Union of reference-data keys across all of this eval's V2 configs.
1359
1360        Returns deduplicated keys in stable insertion order.
1361        """
1362        seen: set[str] = set()
1363        result: list[str] = []
1364        for config in self.configs(readonly=True):
1365            if config.config_type != EvalConfigType.v2:
1366                continue
1367            if not isinstance(config.properties, V2_PROPERTY_TYPES):
1368                continue
1369            for key in reference_data_keys(config.properties):  # type: ignore[arg-type]
1370                if key not in seen:
1371                    seen.add(key)
1372                    result.append(key)
1373        return result
1374
1375    @model_validator(mode="after")
1376    def upgrade_old_reference_answer_eval_config(self) -> Self:
1377        """
1378        Migration: Set the first judge config as the default for existing reference answer evals that don't have a current_config_id set.
1379
1380        For reference_answer evals that don't have a current_config_id set, this migration
1381        will set the first config (by created_at) as the default.
1382        """
1383        if self.id is None:
1384            return self
1385
1386        # Only run during file loading
1387        if not self._loaded_from_file:
1388            return self
1389
1390        # Skip if already migrated (has a current_config_id set)
1391        if self.current_config_id is not None:
1392            return self
1393
1394        # Only migrate reference_answer evals
1395        if self.evaluation_data_type != EvalDataType.reference_answer:
1396            return self
1397
1398        # Prevent recursion: self.configs() loads child files, which re-loads this parent
1399        # (see basemodel.py where we iterate_children_paths_of_parent_path calls load_from_file)
1400        # This causes the validator to run again, creating an infinite loop without this guard.
1401        with _migration_lock:
1402            if self.id in _currently_migrating_eval_ids:
1403                return self
1404            _currently_migrating_eval_ids.add(self.id)
1405
1406        try:
1407            # Get the configs - these are loaded from child files
1408            configs_list = self.configs(readonly=True)
1409            if configs_list and len(configs_list) > 0:
1410                # Sort by created_at to get the oldest (first created) config
1411                sorted_configs = sorted(configs_list, key=lambda c: c.created_at)
1412                self.current_config_id = sorted_configs[0].id
1413        finally:
1414            with _migration_lock:
1415                _currently_migrating_eval_ids.discard(self.id)
1416
1417        return self
1418
1419    @model_validator(mode="after")
1420    def validate_scores(self) -> Self:
1421        if self.output_scores is None or len(self.output_scores) == 0:
1422            raise ValueError(
1423                "output_scores are required, and must have at least one score."
1424            )
1425
1426        # check for duplicate names (once transformed to JSON keys)
1427        output_score_keys = [score.json_key() for score in self.output_scores]
1428        if len(output_score_keys) != len(set(output_score_keys)):
1429            raise ValueError(
1430                f"output_scores must have unique names (once transformed to JSON keys). Got: [{', '.join(output_score_keys)}]"
1431            )
1432        return self
1433
1434    @model_validator(mode="after")
1435    def validate_template_properties(self) -> Self:
1436        if self.template is None:
1437            return self
1438
1439        if (
1440            self.template is not EvalTemplateId.rag
1441            and self.eval_configs_filter_id is None
1442        ):
1443            raise ValueError(
1444                "eval_configs_filter_id is required for all templates except 'rag'"
1445            )
1446
1447        # For spec-based evals, template_properties will be None and validation happens in the spec
1448        # For legacy evals, template_properties contains the data and we validate here
1449        if self.template_properties is None:
1450            return self
1451
1452        # Check for properties that are required for the issue template (legacy evals only)
1453        if self.template == EvalTemplateId.issue:
1454            if "issue_prompt" not in self.template_properties or not isinstance(
1455                self.template_properties["issue_prompt"], str
1456            ):
1457                raise ValueError("issue_prompt is required for issue template")
1458            if "failure_example" in self.template_properties and not isinstance(
1459                self.template_properties["failure_example"], str
1460            ):
1461                raise ValueError(
1462                    "failure_example is optional for issue template, but if provided must be a string"
1463                )
1464            if "pass_example" in self.template_properties and not isinstance(
1465                self.template_properties["pass_example"], str
1466            ):
1467                raise ValueError(
1468                    "pass_example is optional for issue template, but if provided must be a string"
1469                )
1470
1471        if self.template == EvalTemplateId.tool_call:
1472            if self.evaluation_data_type != EvalDataType.full_trace:
1473                raise ValueError(
1474                    "tool_call template should have evaluation_data_type set to full_trace"
1475                )
1476            if (
1477                "tool" not in self.template_properties
1478                or not isinstance(self.template_properties["tool"], str)
1479                or not self.template_properties["tool"].strip()
1480            ):
1481                raise ValueError("tool is required for tool call template")
1482            if "tool_function_name" not in self.template_properties or not isinstance(
1483                self.template_properties["tool_function_name"], str
1484            ):
1485                raise ValueError(
1486                    "tool_function_name is required for tool call template"
1487                )
1488            if (
1489                "appropriate_tool_use_guidelines" not in self.template_properties
1490                or not isinstance(
1491                    self.template_properties["appropriate_tool_use_guidelines"], str
1492                )
1493                or not self.template_properties[
1494                    "appropriate_tool_use_guidelines"
1495                ].strip()
1496            ):
1497                raise ValueError(
1498                    "appropriate_tool_use_guidelines is required for tool call template"
1499                )
1500            if (
1501                "inappropriate_tool_use_guidelines" in self.template_properties
1502                and not isinstance(
1503                    self.template_properties["inappropriate_tool_use_guidelines"], str
1504                )
1505            ):
1506                raise ValueError(
1507                    "inappropriate_tool_use_guidelines is optional for tool call template, but if provided must be a string"
1508                )
1509        return self