kiln_ai.adapters.eval.registry

View Source

 1from typing import TYPE_CHECKING
 2
 3from kiln_ai.adapters.eval.base_eval import BaseEval, BaseV2EvalBridge
 4
 5if TYPE_CHECKING:
 6    from kiln_ai.adapters.model_adapters.base_adapter import SkillsDict
 7    from kiln_ai.datamodel.task import RunConfigProperties
 8from kiln_ai.adapters.eval.g_eval import GEval
 9from kiln_ai.adapters.eval.v2_eval_code_eval import CodeEvalAdapter
10from kiln_ai.adapters.eval.v2_eval_contains import ContainsEval
11from kiln_ai.adapters.eval.v2_eval_exact_match import ExactMatchEval
12from kiln_ai.adapters.eval.v2_eval_llm_judge import LlmJudgeEval
13from kiln_ai.adapters.eval.v2_eval_pattern_match import PatternMatchEval
14from kiln_ai.adapters.eval.v2_eval_set_check import SetCheckEval
15from kiln_ai.adapters.eval.v2_eval_step_count_check import StepCountCheckEval
16from kiln_ai.adapters.eval.v2_eval_tool_call_check import ToolCallCheckEval
17from kiln_ai.datamodel.eval import (
18    V2_PROPERTY_TYPES,
19    EvalConfig,
20    EvalConfigType,
21    V2EvalType,
22)
23from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
24
25_V2_ADAPTER_MAP: dict[V2EvalType, type[BaseV2EvalBridge]] = {
26    V2EvalType.exact_match: ExactMatchEval,
27    V2EvalType.pattern_match: PatternMatchEval,
28    V2EvalType.contains: ContainsEval,
29    V2EvalType.set_check: SetCheckEval,
30    V2EvalType.tool_call_check: ToolCallCheckEval,
31    V2EvalType.step_count_check: StepCountCheckEval,
32    V2EvalType.llm_judge: LlmJudgeEval,
33    V2EvalType.code_eval: CodeEvalAdapter,
34}
35
36
37def legacy_eval_adapter_from_type(eval_config: EvalConfig) -> type[BaseEval]:
38    """Legacy dispatch -- returns a BaseEval subclass for g_eval/llm_as_judge.
39
40    For v2, raises NotImplementedError (v2 adapters use v2_eval_adapter_from_config).
41    """
42    match eval_config.config_type:
43        case EvalConfigType.g_eval:
44            return GEval
45        case EvalConfigType.llm_as_judge:
46            return GEval
47        case EvalConfigType.v2:
48            raise NotImplementedError(
49                "V2 eval configs should use v2_eval_adapter_from_config(), not legacy_eval_adapter_from_type()"
50            )
51        case _:
52            raise_exhaustive_enum_error(eval_config.config_type)
53
54
55def v2_eval_adapter_from_config(
56    eval_config: EvalConfig,
57    run_config: "RunConfigProperties | None" = None,
58    skills: "SkillsDict | None" = None,
59) -> BaseV2EvalBridge:
60    """V2 dispatch -- reads properties.type and looks up the adapter in _V2_ADAPTER_MAP.
61
62    Returns an instantiated adapter, or raises NotImplementedError if the
63    V2 type has no registered adapter (type_not_available skip path).
64    """
65    if eval_config.config_type != EvalConfigType.v2:
66        raise ValueError("v2_eval_adapter_from_config only accepts V2 configs")
67    if not isinstance(eval_config.properties, V2_PROPERTY_TYPES):
68        raise ValueError("V2 config must have typed properties")
69    v2_type = eval_config.properties.type  # type: ignore[union-attr]
70    adapter_cls = _V2_ADAPTER_MAP.get(v2_type)
71    if adapter_cls is None:
72        raise NotImplementedError(f"V2 eval type '{v2_type}' is not yet implemented")
73    return adapter_cls(eval_config, run_config, skills)

def legacy_eval_adapter_from_type( eval_config: kiln_ai.datamodel.eval.EvalConfig) -> type[kiln_ai.adapters.eval.base_eval.BaseEval]: View Source

38def legacy_eval_adapter_from_type(eval_config: EvalConfig) -> type[BaseEval]:
39    """Legacy dispatch -- returns a BaseEval subclass for g_eval/llm_as_judge.
40
41    For v2, raises NotImplementedError (v2 adapters use v2_eval_adapter_from_config).
42    """
43    match eval_config.config_type:
44        case EvalConfigType.g_eval:
45            return GEval
46        case EvalConfigType.llm_as_judge:
47            return GEval
48        case EvalConfigType.v2:
49            raise NotImplementedError(
50                "V2 eval configs should use v2_eval_adapter_from_config(), not legacy_eval_adapter_from_type()"
51            )
52        case _:
53            raise_exhaustive_enum_error(eval_config.config_type)

Legacy dispatch -- returns a BaseEval subclass for g_eval/llm_as_judge.

For v2, raises NotImplementedError (v2 adapters use v2_eval_adapter_from_config).

def v2_eval_adapter_from_config( eval_config: kiln_ai.datamodel.eval.EvalConfig, run_config: Optional[Annotated[Union[Annotated[kiln_ai.datamodel.run_config.KilnAgentRunConfigProperties, Tag(tag='kiln_agent')], Annotated[kiln_ai.datamodel.run_config.McpRunConfigProperties, Tag(tag='mcp')]], Discriminator(discriminator=<function _get_run_config_type>, custom_error_type=None, custom_error_message=None, custom_error_context=None)]] = None, skills: Optional[Dict[str, kiln_ai.datamodel.Skill]] = None) -> kiln_ai.adapters.eval.base_eval.BaseV2EvalBridge: View Source

56def v2_eval_adapter_from_config(
57    eval_config: EvalConfig,
58    run_config: "RunConfigProperties | None" = None,
59    skills: "SkillsDict | None" = None,
60) -> BaseV2EvalBridge:
61    """V2 dispatch -- reads properties.type and looks up the adapter in _V2_ADAPTER_MAP.
62
63    Returns an instantiated adapter, or raises NotImplementedError if the
64    V2 type has no registered adapter (type_not_available skip path).
65    """
66    if eval_config.config_type != EvalConfigType.v2:
67        raise ValueError("v2_eval_adapter_from_config only accepts V2 configs")
68    if not isinstance(eval_config.properties, V2_PROPERTY_TYPES):
69        raise ValueError("V2 config must have typed properties")
70    v2_type = eval_config.properties.type  # type: ignore[union-attr]
71    adapter_cls = _V2_ADAPTER_MAP.get(v2_type)
72    if adapter_cls is None:
73        raise NotImplementedError(f"V2 eval type '{v2_type}' is not yet implemented")
74    return adapter_cls(eval_config, run_config, skills)

V2 dispatch -- reads properties.type and looks up the adapter in _V2_ADAPTER_MAP.

Returns an instantiated adapter, or raises NotImplementedError if the V2 type has no registered adapter (type_not_available skip path).