kiln_ai.datamodel.eval

  1import json
  2from enum import Enum
  3from threading import Lock
  4from typing import TYPE_CHECKING, Any, Dict, List, Union
  5
  6from pydantic import BaseModel, Field, model_validator
  7from typing_extensions import Self
  8
  9from kiln_ai.datamodel.basemodel import (
 10    ID_TYPE,
 11    FilenameString,
 12    FilenameStringShort,
 13    KilnParentedModel,
 14    KilnParentModel,
 15)
 16from kiln_ai.datamodel.datamodel_enums import TaskOutputRatingType
 17from kiln_ai.datamodel.dataset_filters import DatasetFilterId
 18from kiln_ai.datamodel.json_schema import string_to_json_key
 19from kiln_ai.datamodel.task_run import Usage
 20from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 21
 22if TYPE_CHECKING:
 23    from kiln_ai.datamodel.spec import Spec
 24    from kiln_ai.datamodel.task import Task
 25
 26EvalScores = Dict[str, float]
 27
 28# Module-level set to track evals currently being migrated (to prevent recursion)
 29# Protected by _migration_lock to ensure thread-safe access
 30_migration_lock = Lock()
 31_currently_migrating_eval_ids: set[ID_TYPE] = set()
 32
 33
 34class EvalTemplateId(str, Enum):
 35    """
 36    An eval template is a pre-defined eval that can be used as a starting point for a new eval.
 37    """
 38
 39    kiln_requirements = "kiln_requirements"
 40    desired_behaviour = "desired_behaviour"
 41    issue = "kiln_issue"
 42    tool_call = "tool_call"
 43    toxicity = "toxicity"
 44    bias = "bias"
 45    maliciousness = "maliciousness"
 46    factual_correctness = "factual_correctness"
 47    jailbreak = "jailbreak"
 48    rag = "rag"
 49
 50
 51class EvalConfigType(str, Enum):
 52    g_eval = "g_eval"
 53    llm_as_judge = "llm_as_judge"
 54
 55
 56class EvalOutputScore(BaseModel):
 57    """
 58    A definition of a score that an evaluator will produce.
 59
 60    Very similar to TaskRequirement, but conceptually different keeping in a separate models.
 61    """
 62
 63    name: FilenameStringShort = Field(
 64        description="The name of the score. Will be provided to the model so use a descriptive name. Should align to the model's TaskRequirement name if you want to use human evals to evaluate the evaluator's performance."
 65    )
 66    instruction: str | None = Field(
 67        default=None,
 68        description="A description of the score, used to help the model understand the goal of the score. Will be provided to evaluator models, so should be written for the model, not the team/user.",
 69    )
 70    type: TaskOutputRatingType = Field(
 71        description="The type of rating to use ('five_star', 'pass_fail', 'pass_fail_critical').",
 72    )
 73
 74    def json_key(self) -> str:
 75        """
 76        The JSON key for the score, used when running the evaluator with a LLM and we need JSON output.
 77
 78        For example, "Overall Rating" -> "overall_rating"
 79        """
 80        return string_to_json_key(self.name)
 81
 82    @model_validator(mode="after")
 83    def validate_type(self) -> Self:
 84        if self.type == TaskOutputRatingType.custom:
 85            raise ValueError(
 86                f"Custom scores are not supported in evaluators. Score '{self.name}' was set to a custom score."
 87            )
 88        return self
 89
 90
 91class EvalRun(KilnParentedModel):
 92    """
 93    The results of running an eval on a single dataset item.
 94
 95    This is a child of an EvalConfig, which specifies how the scores were generated.
 96
 97    Eval runs can be one of 2 types:
 98    1) eval_config_eval=False: we were evaluating a task run (a method of running the task). We get the task input from the dataset_id.input, run the task with the task_run_config, then ran the evaluator on that output. task_run_config_id must be set. The output saved in this model is the output of the task run.
 99    2) eval_config_eval=True: we were evaluating an eval config (a method of evaluating the task). We used the existing dataset item input/output, and ran the evaluator on it. task_run_config_id must be None. The input/output saved in this model is the input/output of the dataset item.
100    """
101
102    dataset_id: ID_TYPE = Field(
103        description="The ID of the dataset item that was used for this run. Must belong to the same Task as the grand-parent eval of this EvalRun."
104    )
105    task_run_config_id: ID_TYPE | None = Field(
106        description="The ID of the TaskRunConfig that was run, if this eval run was based on a task run. Must belong to the same Task as this eval. Can be None if this eval run is based on an eval config."
107    )
108    eval_config_eval: bool = Field(
109        description="Whether this eval run to evaluate the parent eval config (evaluating the config using an existing dataset item). If true, task_run_config_id must be None, as we're not running the task.",
110        default=False,
111    )
112    # These two may duplicate the dataset_id.input/output, but we're denormalizing intentionally.
113    input: str = Field(
114        description="The input to the task. JSON formatted for structured input, plaintext for unstructured input."
115    )
116    output: str = Field(
117        description="The output of the task. JSON formatted for structured output, plaintext for unstructured output."
118    )
119    reference_answer: str | None = Field(
120        default=None,
121        description="The reference answer for the input. JSON formatted for structured reference answer, plaintext for unstructured reference answer. Used for reference answer evals.",
122    )
123    intermediate_outputs: Dict[str, str] | None = Field(
124        default=None,
125        description="The intermediate outputs of the task (example, eval thinking).",
126    )
127    task_run_trace: str | None = Field(
128        default=None,
129        description="The JSON formatted trace of the task run that produced the output.",
130    )
131    scores: EvalScores = Field(
132        description="The output scores of the evaluator (aligning to those required by the grand-parent Eval this object is a child of)."
133    )
134    task_run_usage: Usage | None = Field(
135        default=None,
136        description="The usage of the task run that produced this eval run output (not the usage by the evaluation model).",
137    )
138
139    def parent_eval_config(self) -> Union["EvalConfig", None]:
140        if self.parent is not None and self.parent.__class__.__name__ != "EvalConfig":
141            raise ValueError("parent must be an EvalConfig")
142        return self.parent  # type: ignore
143
144    @model_validator(mode="after")
145    def validate_output_fields(self) -> Self:
146        parent_eval_config = self.parent_eval_config()
147        parent_eval = parent_eval_config.parent_eval() if parent_eval_config else None
148        if not parent_eval:
149            return self
150
151        evaluation_data_type = parent_eval.evaluation_data_type
152        if (
153            evaluation_data_type == EvalDataType.final_answer
154            and self.task_run_trace is not None
155        ):
156            raise ValueError("final_answer runs should not set trace")
157        elif (
158            not self.eval_config_eval
159            and evaluation_data_type == EvalDataType.full_trace
160            and self.task_run_trace is None
161        ):
162            raise ValueError("full_trace task run eval runs should include trace")
163
164        return self
165
166    @model_validator(mode="after")
167    def validate_eval_run_types(self) -> Self:
168        if self.eval_config_eval and self.task_run_config_id is not None:
169            raise ValueError(
170                "task_run_config_id must be None if eval_config_eval is true"
171            )
172        if not self.eval_config_eval and self.task_run_config_id is None:
173            raise ValueError(
174                "task_run_config_id must be set if eval_config_eval is false"
175            )
176        return self
177
178    @model_validator(mode="after")
179    def validate_scores(self) -> Self:
180        # We're checking the scores have the expected keys from the grand-parent eval
181        if self.scores is None or len(self.scores) == 0:
182            raise ValueError("scores are required, and must have at least one score.")
183
184        parent_eval_config = self.parent_eval_config()
185        eval = parent_eval_config.parent_eval() if parent_eval_config else None
186        if not eval:
187            # Can't validate without the grand-parent eval, allow it to be validated later
188            return self
189
190        output_score_keys = [score.json_key() for score in eval.output_scores]
191        if set(output_score_keys) != set(self.scores.keys()):
192            raise ValueError(
193                f"The scores produced by the evaluator must match the scores expected by the eval. Got: [{', '.join(self.scores.keys())}] and expected: [{', '.join(output_score_keys)}]"
194            )
195
196        # Check that each score is expected in this eval and the correct type
197        for output_score in eval.output_scores:
198            match output_score.type:
199                case TaskOutputRatingType.five_star:
200                    five_star_score = self.scores[output_score.json_key()]
201                    if (
202                        not isinstance(five_star_score, float)
203                        or five_star_score < 1.0
204                        or five_star_score > 5.0
205                    ):
206                        raise ValueError(
207                            f"Score {output_score.name} is a five_star rating and must be a float between 1.0 and 5.0 inclusive. Got: {five_star_score}"
208                        )
209                case TaskOutputRatingType.pass_fail:
210                    pass_fail_score = self.scores[output_score.json_key()]
211                    if (
212                        not isinstance(pass_fail_score, float)
213                        or pass_fail_score < 0.0
214                        or pass_fail_score > 1.0
215                    ):
216                        raise ValueError(
217                            f"Score {output_score.name} is a pass_fail rating and must be a float between 0.0 and 1.0 inclusive. Got: {pass_fail_score}"
218                        )
219                case TaskOutputRatingType.pass_fail_critical:
220                    pass_fail_critical_score = self.scores[output_score.json_key()]
221                    if (
222                        not isinstance(pass_fail_critical_score, float)
223                        or pass_fail_critical_score < -1.0
224                        or pass_fail_critical_score > 1.0
225                    ):
226                        raise ValueError(
227                            f"Score {output_score.name} is a pass_fail_critical rating and must be a float between -1.0 and 1.0 inclusive. Got: {pass_fail_critical_score}"
228                        )
229                case TaskOutputRatingType.custom:
230                    raise ValueError(
231                        f"Custom scores are not supported in evaluators. '{output_score.name}' was set to a custom score."
232                    )
233                case _:
234                    # Catch missing cases
235                    raise_exhaustive_enum_error(output_score.type)
236        return self
237
238    @model_validator(mode="after")
239    def validate_reference_answer(self) -> Self:
240        parent_eval_config = self.parent_eval_config()
241        parent_eval = parent_eval_config.parent_eval() if parent_eval_config else None
242        if not parent_eval:
243            # Can't validate without the grand-parent eval, allow it to be validated later
244            return self
245
246        evaluation_data_type = parent_eval.evaluation_data_type
247        if (
248            self.reference_answer is not None
249            and evaluation_data_type != EvalDataType.reference_answer
250        ):
251            raise ValueError(
252                f"reference_answer is only valid for reference answer evals. Got: {evaluation_data_type.value}"
253            )
254        return self
255
256
257class EvalConfig(KilnParentedModel, KilnParentModel, parent_of={"runs": EvalRun}):
258    """
259    A configuration for running an eval. This includes anything needed to run the eval on a dataset like the prompt, model, thresholds, etc.
260
261    A eval might have many configs, example running the same eval with 2 different models. Comparing eval results is only valid within the scope of the same config.
262    """
263
264    name: FilenameString = Field(description="The name of the eval config.")
265    model_name: str = Field(
266        description="The name of the model to use for this eval config. ",
267    )
268    model_provider: str = Field(
269        description="The provider of the model to use for this eval config.",
270    )
271    config_type: EvalConfigType = Field(
272        default=EvalConfigType.g_eval,
273        description="This is used to determine the type of eval to run.",
274    )
275    properties: dict[str, Any] = Field(
276        default={},
277        description="Properties to be used to execute the eval config. This is config_type specific and should serialize to a json dict.",
278    )
279
280    def parent_eval(self) -> Union["Eval", None]:
281        if self.parent is not None and self.parent.__class__.__name__ != "Eval":
282            raise ValueError("parent must be an Eval")
283        return self.parent  # type: ignore
284
285    def runs(self, readonly: bool = False) -> list[EvalRun]:
286        return super().runs(readonly=readonly)  # type: ignore
287
288    @model_validator(mode="after")
289    def validate_properties(self) -> Self:
290        if (
291            self.config_type == EvalConfigType.g_eval
292            or self.config_type == EvalConfigType.llm_as_judge
293        ):
294            if "eval_steps" not in self.properties or not isinstance(
295                self.properties["eval_steps"], list
296            ):
297                raise ValueError("eval_steps is required and must be a list for g_eval")
298            if "task_description" in self.properties and not isinstance(
299                self.properties["task_description"], str
300            ):
301                raise ValueError(
302                    "task_description is optional, but if provided must be a string"
303                )
304            return self
305        else:
306            raise ValueError(f"Invalid eval config type: {self.config_type}")
307
308    @model_validator(mode="after")
309    def validate_json_serializable(self) -> "EvalConfig":
310        try:
311            # This will raise a TypeError if the dict contains non-JSON-serializable objects
312            json.dumps(self.properties)
313        except TypeError as e:
314            raise ValueError(f"Properties must be JSON serializable: {e!s}")
315        return self
316
317
318class EvalDataType(str, Enum):
319    final_answer = "final_answer"
320    full_trace = "full_trace"
321    reference_answer = "reference_answer"
322
323
324class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}):
325    name: FilenameString = Field(description="The name of the eval.")
326    description: str | None = Field(
327        default=None, description="The description of the eval"
328    )
329    template: EvalTemplateId | None = Field(
330        default=None,
331        description="The template selected when creating this eval. Useful for suggesting eval steps and output scores.",
332    )
333    current_config_id: ID_TYPE = Field(
334        default=None,
335        description="The id of the current config to use for this eval. This can be changed over time to run the same eval with different configs.",
336    )
337    eval_set_filter_id: DatasetFilterId = Field(
338        description="The id of the dataset filter which defines which dataset items are included when running this eval. Should be mutually exclusive with eval_configs_filter_id and train_set_filter_id."
339    )
340    eval_configs_filter_id: DatasetFilterId | None = Field(
341        default=None,
342        description="The id of the dataset filter which defines which dataset items are included when comparing the quality of the eval configs under this eval. Should consist of dataset items with ratings. Should be mutually exclusive with eval_set_filter_id.",
343    )
344    train_set_filter_id: DatasetFilterId | None = Field(
345        default=None,
346        description="The id of the dataset filter which defines which dataset items are included in the training set for fine-tuning. Should be mutually exclusive with eval_set_filter_id.",
347    )
348    output_scores: List[EvalOutputScore] = Field(
349        description="The scores this evaluator should produce."
350    )
351    favourite: bool = Field(
352        default=False,
353        description="Whether this eval is a favourite of the user. Rendered as a star icon in the UI.",
354    )
355    template_properties: dict[str, str | int | bool | float] | None = Field(
356        default=None,
357        description="Properties to be used to execute the eval. This is template_type specific and should serialize to a json dict.",
358    )
359    evaluation_data_type: EvalDataType = Field(
360        default=EvalDataType.final_answer,
361        description="The output of the task run to evaluate. Can be final answer or full trace.",
362    )
363
364    # Workaround to return typed parent without importing Task
365    def parent_task(self) -> Union["Task", None]:
366        if self.parent is not None and self.parent.__class__.__name__ != "Task":
367            raise ValueError("parent must be a Task")
368        return self.parent  # type: ignore
369
370    def configs(self, readonly: bool = False) -> list[EvalConfig]:
371        return super().configs(readonly=readonly)  # type: ignore
372
373    # Workaround to return typed parent without importing Spec
374    def associated_spec(self, readonly: bool = False) -> Union["Spec", None]:
375        """
376        Get the spec associated with this eval, if any.
377        Returns None for legacy evals that are not associated with a spec.
378        """
379
380        task = self.parent_task()
381        if not task or not self.id:
382            return None
383
384        specs = task.specs(readonly=readonly)
385        for spec in specs:
386            if spec.eval_id == self.id:
387                return spec
388        return None
389
390    @model_validator(mode="after")
391    def upgrade_old_reference_answer_eval_config(self) -> Self:
392        """
393        Migration: Set the first judge config as the default for existing reference answer evals that don't have a current_config_id set.
394
395        For reference_answer evals that don't have a current_config_id set, this migration
396        will set the first config (by created_at) as the default.
397        """
398        if self.id is None:
399            return self
400
401        # Only run during file loading
402        if not self._loaded_from_file:
403            return self
404
405        # Skip if already migrated (has a current_config_id set)
406        if self.current_config_id is not None:
407            return self
408
409        # Only migrate reference_answer evals
410        if self.evaluation_data_type != EvalDataType.reference_answer:
411            return self
412
413        # Prevent recursion: self.configs() loads child files, which re-loads this parent
414        # (see basemodel.py where we iterate_children_paths_of_parent_path calls load_from_file)
415        # This causes the validator to run again, creating an infinite loop without this guard.
416        with _migration_lock:
417            if self.id in _currently_migrating_eval_ids:
418                return self
419            _currently_migrating_eval_ids.add(self.id)
420
421        try:
422            # Get the configs - these are loaded from child files
423            configs_list = self.configs(readonly=True)
424            if configs_list and len(configs_list) > 0:
425                # Sort by created_at to get the oldest (first created) config
426                sorted_configs = sorted(configs_list, key=lambda c: c.created_at)
427                self.current_config_id = sorted_configs[0].id
428        finally:
429            with _migration_lock:
430                _currently_migrating_eval_ids.discard(self.id)
431
432        return self
433
434    @model_validator(mode="after")
435    def validate_scores(self) -> Self:
436        if self.output_scores is None or len(self.output_scores) == 0:
437            raise ValueError(
438                "output_scores are required, and must have at least one score."
439            )
440
441        # check for duplicate names (once transformed to JSON keys)
442        output_score_keys = [score.json_key() for score in self.output_scores]
443        if len(output_score_keys) != len(set(output_score_keys)):
444            raise ValueError(
445                f"output_scores must have unique names (once transformed to JSON keys). Got: [{', '.join(output_score_keys)}]"
446            )
447        return self
448
449    @model_validator(mode="after")
450    def validate_template_properties(self) -> Self:
451        # eval_configs_filter_id is required for all templates except "rag"
452        if (
453            self.template is not EvalTemplateId.rag
454            and self.eval_configs_filter_id is None
455        ):
456            raise ValueError(
457                "eval_configs_filter_id is required for all templates except 'rag'"
458            )
459
460        # For spec-based evals, template_properties will be None and validation happens in the spec
461        # For legacy evals, template_properties contains the data and we validate here
462        if self.template_properties is None:
463            return self
464
465        # Check for properties that are required for the issue template (legacy evals only)
466        if self.template == EvalTemplateId.issue:
467            if "issue_prompt" not in self.template_properties or not isinstance(
468                self.template_properties["issue_prompt"], str
469            ):
470                raise ValueError("issue_prompt is required for issue template")
471            if "failure_example" in self.template_properties and not isinstance(
472                self.template_properties["failure_example"], str
473            ):
474                raise ValueError(
475                    "failure_example is optional for issue template, but if provided must be a string"
476                )
477            if "pass_example" in self.template_properties and not isinstance(
478                self.template_properties["pass_example"], str
479            ):
480                raise ValueError(
481                    "pass_example is optional for issue template, but if provided must be a string"
482                )
483
484        if self.template == EvalTemplateId.tool_call:
485            if self.evaluation_data_type != EvalDataType.full_trace:
486                raise ValueError(
487                    "tool_call template should have evaluation_data_type set to full_trace"
488                )
489            if (
490                "tool" not in self.template_properties
491                or not isinstance(self.template_properties["tool"], str)
492                or not self.template_properties["tool"].strip()
493            ):
494                raise ValueError("tool is required for tool call template")
495            if "tool_function_name" not in self.template_properties or not isinstance(
496                self.template_properties["tool_function_name"], str
497            ):
498                raise ValueError(
499                    "tool_function_name is required for tool call template"
500                )
501            if (
502                "appropriate_tool_use_guidelines" not in self.template_properties
503                or not isinstance(
504                    self.template_properties["appropriate_tool_use_guidelines"], str
505                )
506                or not self.template_properties[
507                    "appropriate_tool_use_guidelines"
508                ].strip()
509            ):
510                raise ValueError(
511                    "appropriate_tool_use_guidelines is required for tool call template"
512                )
513            if (
514                "inappropriate_tool_use_guidelines" in self.template_properties
515                and not isinstance(
516                    self.template_properties["inappropriate_tool_use_guidelines"], str
517                )
518            ):
519                raise ValueError(
520                    "inappropriate_tool_use_guidelines is optional for tool call template, but if provided must be a string"
521                )
522        return self