kiln_ai.adapters.eval.v2_eval_contains

View Source

 1from kiln_ai.adapters.eval.base_eval import BaseV2EvalBridge
 2from kiln_ai.adapters.eval.eval_utils.v2_eval_helpers import (
 3    build_binary_scores,
 4    check_reference_key,
 5    extract_output_value,
 6    stringify_for_match,
 7)
 8from kiln_ai.datamodel.eval import (
 9    ContainsProperties,
10    EvalTaskInput,
11    V2EvalResult,
12)
13
14
15class ContainsEval(BaseV2EvalBridge):
16    """V2 adapter for contains: checks whether the extracted value contains a substring."""
17
18    async def evaluate(self, eval_input: EvalTaskInput) -> V2EvalResult:
19        props = self.properties
20        assert isinstance(props, ContainsProperties)
21
22        value, fail_result = extract_output_value(
23            props.value_expression, eval_input, self._output_scores
24        )
25        if fail_result is not None:
26            return fail_result
27
28        if props.reference_key is not None:
29            substring, skip, detail = check_reference_key(
30                props.reference_key, eval_input
31            )
32            if skip is not None:
33                return V2EvalResult(skipped_reason=skip, skipped_detail=detail)
34            substring = stringify_for_match(substring)
35        else:
36            substring = props.substring
37            assert substring is not None
38
39        actual = stringify_for_match(value)
40        if props.case_sensitive:
41            found = substring in actual
42        else:
43            found = substring.lower() in actual.lower()
44
45        if props.mode == "must_contain":
46            passed = found
47        else:
48            passed = not found
49
50        return V2EvalResult(
51            scores=build_binary_scores(self._output_scores, passed),
52        )

class ContainsEval(kiln_ai.adapters.eval.base_eval.BaseV2EvalBridge): View Source

16class ContainsEval(BaseV2EvalBridge):
17    """V2 adapter for contains: checks whether the extracted value contains a substring."""
18
19    async def evaluate(self, eval_input: EvalTaskInput) -> V2EvalResult:
20        props = self.properties
21        assert isinstance(props, ContainsProperties)
22
23        value, fail_result = extract_output_value(
24            props.value_expression, eval_input, self._output_scores
25        )
26        if fail_result is not None:
27            return fail_result
28
29        if props.reference_key is not None:
30            substring, skip, detail = check_reference_key(
31                props.reference_key, eval_input
32            )
33            if skip is not None:
34                return V2EvalResult(skipped_reason=skip, skipped_detail=detail)
35            substring = stringify_for_match(substring)
36        else:
37            substring = props.substring
38            assert substring is not None
39
40        actual = stringify_for_match(value)
41        if props.case_sensitive:
42            found = substring in actual
43        else:
44            found = substring.lower() in actual.lower()
45
46        if props.mode == "must_contain":
47            passed = found
48        else:
49            passed = not found
50
51        return V2EvalResult(
52            scores=build_binary_scores(self._output_scores, passed),
53        )

V2 adapter for contains: checks whether the extracted value contains a substring.

async def evaluate( self, eval_input: kiln_ai.datamodel.eval.EvalTaskInput) -> kiln_ai.datamodel.eval.V2EvalResult: View Source

19    async def evaluate(self, eval_input: EvalTaskInput) -> V2EvalResult:
20        props = self.properties
21        assert isinstance(props, ContainsProperties)
22
23        value, fail_result = extract_output_value(
24            props.value_expression, eval_input, self._output_scores
25        )
26        if fail_result is not None:
27            return fail_result
28
29        if props.reference_key is not None:
30            substring, skip, detail = check_reference_key(
31                props.reference_key, eval_input
32            )
33            if skip is not None:
34                return V2EvalResult(skipped_reason=skip, skipped_detail=detail)
35            substring = stringify_for_match(substring)
36        else:
37            substring = props.substring
38            assert substring is not None
39
40        actual = stringify_for_match(value)
41        if props.case_sensitive:
42            found = substring in actual
43        else:
44            found = substring.lower() in actual.lower()
45
46        if props.mode == "must_contain":
47            passed = found
48        else:
49            passed = not found
50
51        return V2EvalResult(
52            scores=build_binary_scores(self._output_scores, passed),
53        )

Inherited Members

kiln_ai.adapters.eval.base_eval.BaseV2EvalBridge: BaseV2EvalBridge; properties; run_eval
kiln_ai.adapters.eval.base_eval.BaseEval: eval_config; eval; target_task; score_schema; run_config; skills; model_and_provider; run_task; run_task_and_eval; build_score_schema