kiln_ai.adapters.eval.g_eval
1from typing import Dict, List, Tuple 2 3from litellm.types.utils import ChatCompletionTokenLogprob 4 5from kiln_ai.adapters.adapter_registry import adapter_for_task 6from kiln_ai.adapters.eval.base_eval import BaseEval 7from kiln_ai.adapters.eval.eval_utils.eval_trace_formatter import EvalTraceFormatter 8from kiln_ai.adapters.eval.eval_utils.eval_utils import EvalUtils 9from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 10 g_eval_single_metric as _g_eval_single_metric, 11) 12from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 13 metric_offsets as _metric_offsets, 14) 15from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 16 rating_token_to_score as _rating_token_to_score, 17) 18from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 19 raw_output_from_logprobs as _raw_output_from_logprobs, 20) 21from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 22 score_from_token_string as _score_from_token_string, 23) 24from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 25 token_search_range as _token_search_range, 26) 27from kiln_ai.adapters.ml_model_list import ( 28 default_structured_output_mode_for_model_provider, 29) 30from kiln_ai.adapters.model_adapters.base_adapter import ( 31 AdapterConfig, 32 RunOutput, 33 SkillsDict, 34) 35from kiln_ai.adapters.prompt_builders import PromptGenerators 36from kiln_ai.datamodel import Project, Task, TaskRun 37from kiln_ai.datamodel.eval import EvalConfig, EvalConfigType, EvalDataType, EvalScores 38from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties 39from kiln_ai.datamodel.task import RunConfigProperties, StructuredOutputMode 40 41 42class GEvalTask(Task, parent_of={}): 43 """ 44 Kiln task for executing a G-Eval. Can be run on any Kiln adapter which supports logprobs. 45 46 Note G-Eval implements both G-Eval and LLM as Judge as they are very similar. 47 """ 48 49 def __init__(self, eval_config: EvalConfig): 50 tmp_project = Project(name="GEval") 51 52 # Build a simple LLM as Judge system instruction 53 system_instruction = "Your job to evaluate a model's performance on a task. Blocks will be marked with <eval_data> tags.\n" 54 # Optionally add a short task description 55 task_description = eval_config.properties.get("task_description", None) 56 if task_description: 57 system_instruction += f"\nThe task the model was given is as follows:\n<eval_data>\n<task_description>{task_description}</task_description>\n</eval_data>\n" 58 59 # Build the COT eval instructions 60 steps = eval_config.properties.get("eval_steps", []) 61 if not isinstance(steps, list): 62 raise ValueError("eval_steps must be a list.") 63 if len(steps) == 1: 64 cot_instructions = "First, think step by step about the model's performance following this evaluation step:\n\n" 65 cot_instructions += f"{steps[0]}\n" 66 else: 67 cot_instructions = "First, think step by step about the model's performance following these evaluation steps:\n\n" 68 for i, step in enumerate(steps): 69 cot_instructions += f"{i + 1}) {step}\n" 70 71 eval = eval_config.parent_eval() 72 if not eval: 73 raise ValueError("Eval config must have a parent eval") 74 75 # Build the output schema from the eval's target output scores. 76 # We restrict the LLM's output scoring schema to discrete scores (pass/fail/critical/1-5) - allow_float_scores=False 77 # However, the final scores from the evaluator can be a float (see later logprob calculation, which requires discrete token outputs) 78 output_schema = BaseEval.build_score_schema(eval, allow_float_scores=False) 79 80 super().__init__( 81 name="GEval Task", 82 parent=tmp_project, 83 instruction=system_instruction, 84 thinking_instruction=cot_instructions, 85 output_json_schema=output_schema, 86 ) 87 88 89class GEval(BaseEval): 90 """ 91 A evaluator which implements G-Eval and LLM as Judge. 92 93 G-Eval is a method of evaluating the quality of a model's output. It is a weighted average of the scores of the tokens in the output. The weights are the log probabilities of the tokens in the output. https://arxiv.org/abs/2303.16634 94 95 LLM as Judge is a method of evaluating the quality of a model's output. It simply asks the LLM to score, and uses the returned output (no logprobs needed). Also called direct evaluation. 96 97 @misc{liu2023gevalnlgevaluationusing, 98 title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment}, 99 author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu}, 100 year={2023}, 101 eprint={2303.16634}, 102 archivePrefix={arXiv}, 103 primaryClass={cs.CL}, 104 url={https://arxiv.org/abs/2303.16634}, 105 } 106 """ 107 108 def __init__( 109 self, 110 eval_config: EvalConfig, 111 run_config: RunConfigProperties | None, 112 skills: SkillsDict | None = None, 113 ): 114 if ( 115 eval_config.config_type != EvalConfigType.g_eval 116 and eval_config.config_type != EvalConfigType.llm_as_judge 117 ): 118 raise ValueError( 119 f"GEval must be initialized with a GEval or LLM as Judge config_type. Got {eval_config.config_type}" 120 ) 121 122 super().__init__(eval_config, run_config, skills=skills) 123 124 self.geval_task = GEvalTask(eval_config) 125 126 def generate_final_answer_run_description( 127 self, eval_input: str, eval_output: str 128 ) -> str: 129 return f"""The model was given the following input for the task: 130<eval_data> 131{eval_input} 132</eval_data> 133 134The model produced the following output for the task: 135<eval_data> 136{eval_output} 137</eval_data> 138""" 139 140 def generate_ref_ans_run_description( 141 self, eval_input: str, eval_output: str, reference_answer: str 142 ) -> str: 143 return f"""The model was given the following input for the task: 144<eval_data> 145{eval_input} 146</eval_data> 147 148The model produced the following output for the task: 149<eval_data> 150{eval_output} 151</eval_data> 152 153This is the reference answer: 154<eval_data> 155{reference_answer} 156</eval_data> 157""" 158 159 def generate_full_trace_run_description( 160 self, 161 eval_input: str, 162 available_tools: str | None, 163 conversation_history: str, 164 ) -> str: 165 description = "" 166 description += f"""The model was given the following <user_input> for the <task_description>: 167<eval_data> 168<user_input>{eval_input}</user_input> 169</eval_data> 170""" 171 # Get properties from spec if available, otherwise from eval.template_properties (for legacy evals) 172 spec = self.eval.associated_spec(readonly=True) 173 174 # Spec uses different keys than legacy eval template_properties 175 if spec: 176 # Spec: tool_use_guidelines, appropriate_tool_use_examples, inappropriate_tool_use_examples 177 tool_use_guidelines = str(spec.properties.get("tool_use_guidelines") or "") 178 appropriate_tool_use_examples = str( 179 spec.properties.get("appropriate_tool_use_examples") or "" 180 ) 181 inappropriate_tool_use_examples = str( 182 spec.properties.get("inappropriate_tool_use_examples") or "" 183 ) 184 description += f"""The model was given the following <tool_use_guidelines>: 185<eval_data> 186<tool_use_guidelines> 187{tool_use_guidelines} 188</tool_use_guidelines> 189</eval_data> 190""" 191 description += f"""The model was given the following <appropriate_tool_use_examples>: 192<eval_data> 193<appropriate_tool_use_examples> 194{appropriate_tool_use_examples} 195</appropriate_tool_use_examples> 196</eval_data> 197""" 198 description += f"""The model was given the following <inappropriate_tool_use_examples>: 199<eval_data> 200<inappropriate_tool_use_examples> 201{inappropriate_tool_use_examples} 202</inappropriate_tool_use_examples> 203</eval_data> 204""" 205 elif self.eval.template_properties: 206 # Legacy eval: appropriate_tool_use_guidelines, inappropriate_tool_use_guidelines 207 appropriate_tool_use_guidelines = str( 208 self.eval.template_properties.get("appropriate_tool_use_guidelines") 209 or "" 210 ) 211 inappropriate_tool_use_guidelines = str( 212 self.eval.template_properties.get("inappropriate_tool_use_guidelines") 213 or "" 214 ) 215 216 description += f"""The model was given the following <appropriate_tool_use_guidelines> guidelines: 217<eval_data> 218<appropriate_tool_use_guidelines> 219{appropriate_tool_use_guidelines} 220</appropriate_tool_use_guidelines> 221</eval_data> 222""" 223 # Only include if it has content since it is optional 224 if inappropriate_tool_use_guidelines: 225 description += f"""The model was given the following <inappropriate_tool_use_guidelines> guidelines: 226<eval_data> 227<inappropriate_tool_use_guidelines> 228{inappropriate_tool_use_guidelines} 229</inappropriate_tool_use_guidelines> 230</eval_data> 231""" 232 233 if available_tools is not None: 234 if available_tools != "": 235 description += f""" 236This is the list of tools available to the model: 237<eval_data> 238<available_tools>{available_tools}</available_tools> 239</eval_data> 240""" 241 else: 242 description += """ 243There were no tools available to the model. 244""" 245 246 description += f""" 247This is the full conversation history for the task run: 248<eval_data> 249<conversation_history>{conversation_history}</conversation_history> 250</eval_data> 251""" 252 return description 253 254 async def run_eval( 255 self, task_run: TaskRun, eval_job_item: TaskRun | None = None 256 ) -> tuple[EvalScores, Dict[str, str] | None]: 257 """ 258 Run this eval on the given task run. 259 """ 260 261 model_name, provider = self.model_and_provider() 262 263 # Only fetch logprobs for G-Eval 264 # There are at most 5 valid rating tokens per rating type (five_star being largest), so 10 is more than enough to get to the very very unlikely 265 top_logprobs = ( 266 10 if self.eval_config.config_type == EvalConfigType.g_eval else None 267 ) 268 269 # We don't expose setting this manually in the UI, so pull a recommended mode from ml_model_list 270 structured_output_mode = default_structured_output_mode_for_model_provider( 271 model_name, 272 provider, 273 default=StructuredOutputMode.json_schema, 274 # G-eval expects JSON, so don't allow function calling modes 275 disallowed_modes=[ 276 StructuredOutputMode.function_calling, 277 StructuredOutputMode.function_calling_weak, 278 ], 279 ) 280 281 adapter = adapter_for_task( 282 self.geval_task, 283 run_config_properties=KilnAgentRunConfigProperties( 284 model_name=model_name, 285 model_provider_name=provider, 286 # We always use Simple COT for G-Eval and LLM as Judge 287 prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT, 288 structured_output_mode=structured_output_mode, 289 ), 290 base_adapter_config=AdapterConfig( 291 # Don't save this run into the task_runs. It will be saved into an eval_run where it belongs 292 allow_saving=False, 293 top_logprobs=top_logprobs, 294 ), 295 ) 296 297 if self.eval.evaluation_data_type == EvalDataType.full_trace: 298 if task_run.trace is None: 299 raise ValueError("Task run trace is required for full trace evaluation") 300 301 available_tools = await EvalUtils.formatted_available_tools_from_task_run( 302 task_run 303 ) 304 run_description = self.generate_full_trace_run_description( 305 task_run.input, 306 available_tools, 307 EvalTraceFormatter.trace_to_formatted_conversation_history( 308 task_run.trace 309 ), 310 ) 311 312 elif self.eval.evaluation_data_type == EvalDataType.reference_answer: 313 if eval_job_item is None: 314 raise ValueError( 315 "Eval job item is required for reference answer evaluation" 316 ) 317 run_description = self.generate_ref_ans_run_description( 318 task_run.input, task_run.output.output, eval_job_item.output.output 319 ) 320 321 else: # EvalDataType.final_answer 322 run_description = self.generate_final_answer_run_description( 323 task_run.input, task_run.output.output 324 ) 325 326 # We don't need the run, but invoke_returning_run_output() runs validations for us over _run() 327 _, run_output = await adapter.invoke_returning_run_output(run_description) 328 329 if self.eval_config.config_type == EvalConfigType.llm_as_judge: 330 return self.build_llm_as_judge_score( 331 run_output 332 ), run_output.intermediate_outputs 333 else: 334 return self.build_g_eval_score(run_output), run_output.intermediate_outputs 335 336 def build_llm_as_judge_score(self, run_output: RunOutput) -> EvalScores: 337 """Build the LLM as Judge score for the given run and run output.""" 338 from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 339 build_llm_as_judge_score, 340 ) 341 342 return build_llm_as_judge_score(run_output, self.score_from_token_string) 343 344 def build_g_eval_score(self, run_output: RunOutput) -> EvalScores: 345 """Build the G-Eval score for the given run and run output. 346 347 We create a weighted average of each rating using the logprobs. 348 349 @misc{liu2023gevalnlgevaluationusing, 350 title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment}, 351 author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu}, 352 year={2023}, 353 eprint={2303.16634}, 354 archivePrefix={arXiv}, 355 primaryClass={cs.CL}, 356 url={https://arxiv.org/abs/2303.16634}, 357 } 358 """ 359 from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 360 build_g_eval_score, 361 ) 362 363 return build_g_eval_score( 364 run_output, 365 self.raw_output_from_logprobs, 366 self.metric_offsets, 367 self.g_eval_single_metric, 368 ) 369 370 def g_eval_single_metric( 371 self, 372 run_output: RunOutput, 373 metric: str, 374 metric_offsets: Dict[str, int], 375 raw_output: str, 376 ) -> float | None: 377 return _g_eval_single_metric(run_output, metric, metric_offsets, raw_output) 378 379 def raw_output_from_logprobs(self, run_output: RunOutput) -> str: 380 return _raw_output_from_logprobs(run_output) 381 382 def token_search_range( 383 self, raw_output: str, metric: str, metric_offsets: Dict[str, int] 384 ) -> Tuple[int, int]: 385 return _token_search_range(raw_output, metric, metric_offsets) 386 387 def rating_token_to_score( 388 self, token_logprob: ChatCompletionTokenLogprob 389 ) -> float | None: 390 return _rating_token_to_score(token_logprob) 391 392 def score_from_token_string(self, token: str) -> float | None: 393 return _score_from_token_string(token) 394 395 def metric_offsets(self, raw_output: str, metrics: List[str]) -> Dict[str, int]: 396 return _metric_offsets(raw_output, metrics)
class
GEvalTask(kiln_ai.datamodel.task.Task):
43class GEvalTask(Task, parent_of={}): 44 """ 45 Kiln task for executing a G-Eval. Can be run on any Kiln adapter which supports logprobs. 46 47 Note G-Eval implements both G-Eval and LLM as Judge as they are very similar. 48 """ 49 50 def __init__(self, eval_config: EvalConfig): 51 tmp_project = Project(name="GEval") 52 53 # Build a simple LLM as Judge system instruction 54 system_instruction = "Your job to evaluate a model's performance on a task. Blocks will be marked with <eval_data> tags.\n" 55 # Optionally add a short task description 56 task_description = eval_config.properties.get("task_description", None) 57 if task_description: 58 system_instruction += f"\nThe task the model was given is as follows:\n<eval_data>\n<task_description>{task_description}</task_description>\n</eval_data>\n" 59 60 # Build the COT eval instructions 61 steps = eval_config.properties.get("eval_steps", []) 62 if not isinstance(steps, list): 63 raise ValueError("eval_steps must be a list.") 64 if len(steps) == 1: 65 cot_instructions = "First, think step by step about the model's performance following this evaluation step:\n\n" 66 cot_instructions += f"{steps[0]}\n" 67 else: 68 cot_instructions = "First, think step by step about the model's performance following these evaluation steps:\n\n" 69 for i, step in enumerate(steps): 70 cot_instructions += f"{i + 1}) {step}\n" 71 72 eval = eval_config.parent_eval() 73 if not eval: 74 raise ValueError("Eval config must have a parent eval") 75 76 # Build the output schema from the eval's target output scores. 77 # We restrict the LLM's output scoring schema to discrete scores (pass/fail/critical/1-5) - allow_float_scores=False 78 # However, the final scores from the evaluator can be a float (see later logprob calculation, which requires discrete token outputs) 79 output_schema = BaseEval.build_score_schema(eval, allow_float_scores=False) 80 81 super().__init__( 82 name="GEval Task", 83 parent=tmp_project, 84 instruction=system_instruction, 85 thinking_instruction=cot_instructions, 86 output_json_schema=output_schema, 87 )
Kiln task for executing a G-Eval. Can be run on any Kiln adapter which supports logprobs.
Note G-Eval implements both G-Eval and LLM as Judge as they are very similar.
GEvalTask(eval_config: kiln_ai.datamodel.eval.EvalConfig)
50 def __init__(self, eval_config: EvalConfig): 51 tmp_project = Project(name="GEval") 52 53 # Build a simple LLM as Judge system instruction 54 system_instruction = "Your job to evaluate a model's performance on a task. Blocks will be marked with <eval_data> tags.\n" 55 # Optionally add a short task description 56 task_description = eval_config.properties.get("task_description", None) 57 if task_description: 58 system_instruction += f"\nThe task the model was given is as follows:\n<eval_data>\n<task_description>{task_description}</task_description>\n</eval_data>\n" 59 60 # Build the COT eval instructions 61 steps = eval_config.properties.get("eval_steps", []) 62 if not isinstance(steps, list): 63 raise ValueError("eval_steps must be a list.") 64 if len(steps) == 1: 65 cot_instructions = "First, think step by step about the model's performance following this evaluation step:\n\n" 66 cot_instructions += f"{steps[0]}\n" 67 else: 68 cot_instructions = "First, think step by step about the model's performance following these evaluation steps:\n\n" 69 for i, step in enumerate(steps): 70 cot_instructions += f"{i + 1}) {step}\n" 71 72 eval = eval_config.parent_eval() 73 if not eval: 74 raise ValueError("Eval config must have a parent eval") 75 76 # Build the output schema from the eval's target output scores. 77 # We restrict the LLM's output scoring schema to discrete scores (pass/fail/critical/1-5) - allow_float_scores=False 78 # However, the final scores from the evaluator can be a float (see later logprob calculation, which requires discrete token outputs) 79 output_schema = BaseEval.build_score_schema(eval, allow_float_scores=False) 80 81 super().__init__( 82 name="GEval Task", 83 parent=tmp_project, 84 instruction=system_instruction, 85 thinking_instruction=cot_instructions, 86 output_json_schema=output_schema, 87 )
Create a new model by parsing and validating input data from keyword arguments.
Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be
validated to form a valid model.
self is explicitly positional-only to allow self as a field name.
model_config =
{'validate_assignment': True}
Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].
def
model_post_init(self: pydantic.main.BaseModel, context: Any, /) -> None:
365def init_private_attributes(self: BaseModel, context: Any, /) -> None: 366 """This function is meant to behave like a BaseModel method to initialize private attributes. 367 368 It takes context as an argument since that's what pydantic-core passes when calling it. 369 370 Args: 371 self: The BaseModel instance. 372 context: The context. 373 """ 374 if getattr(self, '__pydantic_private__', None) is None: 375 pydantic_private = {} 376 for name, private_attr in self.__private_attributes__.items(): 377 # Avoid needlessly creating a new dict for the validated data: 378 if private_attr.default_factory_takes_validated_data: 379 default = private_attr.get_default( 380 call_default_factory=True, validated_data={**self.__dict__, **pydantic_private} 381 ) 382 else: 383 default = private_attr.get_default(call_default_factory=True) 384 if default is not PydanticUndefined: 385 pydantic_private[name] = default 386 object_setattr(self, '__pydantic_private__', pydantic_private)
This function is meant to behave like a BaseModel method to initialize private attributes.
It takes context as an argument since that's what pydantic-core passes when calling it.
Args: self: The BaseModel instance. context: The context.
90class GEval(BaseEval): 91 """ 92 A evaluator which implements G-Eval and LLM as Judge. 93 94 G-Eval is a method of evaluating the quality of a model's output. It is a weighted average of the scores of the tokens in the output. The weights are the log probabilities of the tokens in the output. https://arxiv.org/abs/2303.16634 95 96 LLM as Judge is a method of evaluating the quality of a model's output. It simply asks the LLM to score, and uses the returned output (no logprobs needed). Also called direct evaluation. 97 98 @misc{liu2023gevalnlgevaluationusing, 99 title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment}, 100 author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu}, 101 year={2023}, 102 eprint={2303.16634}, 103 archivePrefix={arXiv}, 104 primaryClass={cs.CL}, 105 url={https://arxiv.org/abs/2303.16634}, 106 } 107 """ 108 109 def __init__( 110 self, 111 eval_config: EvalConfig, 112 run_config: RunConfigProperties | None, 113 skills: SkillsDict | None = None, 114 ): 115 if ( 116 eval_config.config_type != EvalConfigType.g_eval 117 and eval_config.config_type != EvalConfigType.llm_as_judge 118 ): 119 raise ValueError( 120 f"GEval must be initialized with a GEval or LLM as Judge config_type. Got {eval_config.config_type}" 121 ) 122 123 super().__init__(eval_config, run_config, skills=skills) 124 125 self.geval_task = GEvalTask(eval_config) 126 127 def generate_final_answer_run_description( 128 self, eval_input: str, eval_output: str 129 ) -> str: 130 return f"""The model was given the following input for the task: 131<eval_data> 132{eval_input} 133</eval_data> 134 135The model produced the following output for the task: 136<eval_data> 137{eval_output} 138</eval_data> 139""" 140 141 def generate_ref_ans_run_description( 142 self, eval_input: str, eval_output: str, reference_answer: str 143 ) -> str: 144 return f"""The model was given the following input for the task: 145<eval_data> 146{eval_input} 147</eval_data> 148 149The model produced the following output for the task: 150<eval_data> 151{eval_output} 152</eval_data> 153 154This is the reference answer: 155<eval_data> 156{reference_answer} 157</eval_data> 158""" 159 160 def generate_full_trace_run_description( 161 self, 162 eval_input: str, 163 available_tools: str | None, 164 conversation_history: str, 165 ) -> str: 166 description = "" 167 description += f"""The model was given the following <user_input> for the <task_description>: 168<eval_data> 169<user_input>{eval_input}</user_input> 170</eval_data> 171""" 172 # Get properties from spec if available, otherwise from eval.template_properties (for legacy evals) 173 spec = self.eval.associated_spec(readonly=True) 174 175 # Spec uses different keys than legacy eval template_properties 176 if spec: 177 # Spec: tool_use_guidelines, appropriate_tool_use_examples, inappropriate_tool_use_examples 178 tool_use_guidelines = str(spec.properties.get("tool_use_guidelines") or "") 179 appropriate_tool_use_examples = str( 180 spec.properties.get("appropriate_tool_use_examples") or "" 181 ) 182 inappropriate_tool_use_examples = str( 183 spec.properties.get("inappropriate_tool_use_examples") or "" 184 ) 185 description += f"""The model was given the following <tool_use_guidelines>: 186<eval_data> 187<tool_use_guidelines> 188{tool_use_guidelines} 189</tool_use_guidelines> 190</eval_data> 191""" 192 description += f"""The model was given the following <appropriate_tool_use_examples>: 193<eval_data> 194<appropriate_tool_use_examples> 195{appropriate_tool_use_examples} 196</appropriate_tool_use_examples> 197</eval_data> 198""" 199 description += f"""The model was given the following <inappropriate_tool_use_examples>: 200<eval_data> 201<inappropriate_tool_use_examples> 202{inappropriate_tool_use_examples} 203</inappropriate_tool_use_examples> 204</eval_data> 205""" 206 elif self.eval.template_properties: 207 # Legacy eval: appropriate_tool_use_guidelines, inappropriate_tool_use_guidelines 208 appropriate_tool_use_guidelines = str( 209 self.eval.template_properties.get("appropriate_tool_use_guidelines") 210 or "" 211 ) 212 inappropriate_tool_use_guidelines = str( 213 self.eval.template_properties.get("inappropriate_tool_use_guidelines") 214 or "" 215 ) 216 217 description += f"""The model was given the following <appropriate_tool_use_guidelines> guidelines: 218<eval_data> 219<appropriate_tool_use_guidelines> 220{appropriate_tool_use_guidelines} 221</appropriate_tool_use_guidelines> 222</eval_data> 223""" 224 # Only include if it has content since it is optional 225 if inappropriate_tool_use_guidelines: 226 description += f"""The model was given the following <inappropriate_tool_use_guidelines> guidelines: 227<eval_data> 228<inappropriate_tool_use_guidelines> 229{inappropriate_tool_use_guidelines} 230</inappropriate_tool_use_guidelines> 231</eval_data> 232""" 233 234 if available_tools is not None: 235 if available_tools != "": 236 description += f""" 237This is the list of tools available to the model: 238<eval_data> 239<available_tools>{available_tools}</available_tools> 240</eval_data> 241""" 242 else: 243 description += """ 244There were no tools available to the model. 245""" 246 247 description += f""" 248This is the full conversation history for the task run: 249<eval_data> 250<conversation_history>{conversation_history}</conversation_history> 251</eval_data> 252""" 253 return description 254 255 async def run_eval( 256 self, task_run: TaskRun, eval_job_item: TaskRun | None = None 257 ) -> tuple[EvalScores, Dict[str, str] | None]: 258 """ 259 Run this eval on the given task run. 260 """ 261 262 model_name, provider = self.model_and_provider() 263 264 # Only fetch logprobs for G-Eval 265 # There are at most 5 valid rating tokens per rating type (five_star being largest), so 10 is more than enough to get to the very very unlikely 266 top_logprobs = ( 267 10 if self.eval_config.config_type == EvalConfigType.g_eval else None 268 ) 269 270 # We don't expose setting this manually in the UI, so pull a recommended mode from ml_model_list 271 structured_output_mode = default_structured_output_mode_for_model_provider( 272 model_name, 273 provider, 274 default=StructuredOutputMode.json_schema, 275 # G-eval expects JSON, so don't allow function calling modes 276 disallowed_modes=[ 277 StructuredOutputMode.function_calling, 278 StructuredOutputMode.function_calling_weak, 279 ], 280 ) 281 282 adapter = adapter_for_task( 283 self.geval_task, 284 run_config_properties=KilnAgentRunConfigProperties( 285 model_name=model_name, 286 model_provider_name=provider, 287 # We always use Simple COT for G-Eval and LLM as Judge 288 prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT, 289 structured_output_mode=structured_output_mode, 290 ), 291 base_adapter_config=AdapterConfig( 292 # Don't save this run into the task_runs. It will be saved into an eval_run where it belongs 293 allow_saving=False, 294 top_logprobs=top_logprobs, 295 ), 296 ) 297 298 if self.eval.evaluation_data_type == EvalDataType.full_trace: 299 if task_run.trace is None: 300 raise ValueError("Task run trace is required for full trace evaluation") 301 302 available_tools = await EvalUtils.formatted_available_tools_from_task_run( 303 task_run 304 ) 305 run_description = self.generate_full_trace_run_description( 306 task_run.input, 307 available_tools, 308 EvalTraceFormatter.trace_to_formatted_conversation_history( 309 task_run.trace 310 ), 311 ) 312 313 elif self.eval.evaluation_data_type == EvalDataType.reference_answer: 314 if eval_job_item is None: 315 raise ValueError( 316 "Eval job item is required for reference answer evaluation" 317 ) 318 run_description = self.generate_ref_ans_run_description( 319 task_run.input, task_run.output.output, eval_job_item.output.output 320 ) 321 322 else: # EvalDataType.final_answer 323 run_description = self.generate_final_answer_run_description( 324 task_run.input, task_run.output.output 325 ) 326 327 # We don't need the run, but invoke_returning_run_output() runs validations for us over _run() 328 _, run_output = await adapter.invoke_returning_run_output(run_description) 329 330 if self.eval_config.config_type == EvalConfigType.llm_as_judge: 331 return self.build_llm_as_judge_score( 332 run_output 333 ), run_output.intermediate_outputs 334 else: 335 return self.build_g_eval_score(run_output), run_output.intermediate_outputs 336 337 def build_llm_as_judge_score(self, run_output: RunOutput) -> EvalScores: 338 """Build the LLM as Judge score for the given run and run output.""" 339 from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 340 build_llm_as_judge_score, 341 ) 342 343 return build_llm_as_judge_score(run_output, self.score_from_token_string) 344 345 def build_g_eval_score(self, run_output: RunOutput) -> EvalScores: 346 """Build the G-Eval score for the given run and run output. 347 348 We create a weighted average of each rating using the logprobs. 349 350 @misc{liu2023gevalnlgevaluationusing, 351 title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment}, 352 author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu}, 353 year={2023}, 354 eprint={2303.16634}, 355 archivePrefix={arXiv}, 356 primaryClass={cs.CL}, 357 url={https://arxiv.org/abs/2303.16634}, 358 } 359 """ 360 from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 361 build_g_eval_score, 362 ) 363 364 return build_g_eval_score( 365 run_output, 366 self.raw_output_from_logprobs, 367 self.metric_offsets, 368 self.g_eval_single_metric, 369 ) 370 371 def g_eval_single_metric( 372 self, 373 run_output: RunOutput, 374 metric: str, 375 metric_offsets: Dict[str, int], 376 raw_output: str, 377 ) -> float | None: 378 return _g_eval_single_metric(run_output, metric, metric_offsets, raw_output) 379 380 def raw_output_from_logprobs(self, run_output: RunOutput) -> str: 381 return _raw_output_from_logprobs(run_output) 382 383 def token_search_range( 384 self, raw_output: str, metric: str, metric_offsets: Dict[str, int] 385 ) -> Tuple[int, int]: 386 return _token_search_range(raw_output, metric, metric_offsets) 387 388 def rating_token_to_score( 389 self, token_logprob: ChatCompletionTokenLogprob 390 ) -> float | None: 391 return _rating_token_to_score(token_logprob) 392 393 def score_from_token_string(self, token: str) -> float | None: 394 return _score_from_token_string(token) 395 396 def metric_offsets(self, raw_output: str, metrics: List[str]) -> Dict[str, int]: 397 return _metric_offsets(raw_output, metrics)
A evaluator which implements G-Eval and LLM as Judge.
G-Eval is a method of evaluating the quality of a model's output. It is a weighted average of the scores of the tokens in the output. The weights are the log probabilities of the tokens in the output. https://arxiv.org/abs/2303.16634
LLM as Judge is a method of evaluating the quality of a model's output. It simply asks the LLM to score, and uses the returned output (no logprobs needed). Also called direct evaluation.
@misc{liu2023gevalnlgevaluationusing, title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment}, author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu}, year={2023}, eprint={2303.16634}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2303.16634}, }
GEval( eval_config: kiln_ai.datamodel.eval.EvalConfig, run_config: Optional[Annotated[Union[Annotated[kiln_ai.datamodel.run_config.KilnAgentRunConfigProperties, Tag(tag='kiln_agent')], Annotated[kiln_ai.datamodel.run_config.McpRunConfigProperties, Tag(tag='mcp')]], Discriminator(discriminator=<function _get_run_config_type>, custom_error_type=None, custom_error_message=None, custom_error_context=None)]], skills: Optional[Dict[str, kiln_ai.datamodel.Skill]] = None)
109 def __init__( 110 self, 111 eval_config: EvalConfig, 112 run_config: RunConfigProperties | None, 113 skills: SkillsDict | None = None, 114 ): 115 if ( 116 eval_config.config_type != EvalConfigType.g_eval 117 and eval_config.config_type != EvalConfigType.llm_as_judge 118 ): 119 raise ValueError( 120 f"GEval must be initialized with a GEval or LLM as Judge config_type. Got {eval_config.config_type}" 121 ) 122 123 super().__init__(eval_config, run_config, skills=skills) 124 125 self.geval_task = GEvalTask(eval_config)
def
generate_final_answer_run_description(self, eval_input: str, eval_output: str) -> str:
127 def generate_final_answer_run_description( 128 self, eval_input: str, eval_output: str 129 ) -> str: 130 return f"""The model was given the following input for the task: 131<eval_data> 132{eval_input} 133</eval_data> 134 135The model produced the following output for the task: 136<eval_data> 137{eval_output} 138</eval_data> 139"""
def
generate_ref_ans_run_description(self, eval_input: str, eval_output: str, reference_answer: str) -> str:
141 def generate_ref_ans_run_description( 142 self, eval_input: str, eval_output: str, reference_answer: str 143 ) -> str: 144 return f"""The model was given the following input for the task: 145<eval_data> 146{eval_input} 147</eval_data> 148 149The model produced the following output for the task: 150<eval_data> 151{eval_output} 152</eval_data> 153 154This is the reference answer: 155<eval_data> 156{reference_answer} 157</eval_data> 158"""
def
generate_full_trace_run_description( self, eval_input: str, available_tools: str | None, conversation_history: str) -> str:
160 def generate_full_trace_run_description( 161 self, 162 eval_input: str, 163 available_tools: str | None, 164 conversation_history: str, 165 ) -> str: 166 description = "" 167 description += f"""The model was given the following <user_input> for the <task_description>: 168<eval_data> 169<user_input>{eval_input}</user_input> 170</eval_data> 171""" 172 # Get properties from spec if available, otherwise from eval.template_properties (for legacy evals) 173 spec = self.eval.associated_spec(readonly=True) 174 175 # Spec uses different keys than legacy eval template_properties 176 if spec: 177 # Spec: tool_use_guidelines, appropriate_tool_use_examples, inappropriate_tool_use_examples 178 tool_use_guidelines = str(spec.properties.get("tool_use_guidelines") or "") 179 appropriate_tool_use_examples = str( 180 spec.properties.get("appropriate_tool_use_examples") or "" 181 ) 182 inappropriate_tool_use_examples = str( 183 spec.properties.get("inappropriate_tool_use_examples") or "" 184 ) 185 description += f"""The model was given the following <tool_use_guidelines>: 186<eval_data> 187<tool_use_guidelines> 188{tool_use_guidelines} 189</tool_use_guidelines> 190</eval_data> 191""" 192 description += f"""The model was given the following <appropriate_tool_use_examples>: 193<eval_data> 194<appropriate_tool_use_examples> 195{appropriate_tool_use_examples} 196</appropriate_tool_use_examples> 197</eval_data> 198""" 199 description += f"""The model was given the following <inappropriate_tool_use_examples>: 200<eval_data> 201<inappropriate_tool_use_examples> 202{inappropriate_tool_use_examples} 203</inappropriate_tool_use_examples> 204</eval_data> 205""" 206 elif self.eval.template_properties: 207 # Legacy eval: appropriate_tool_use_guidelines, inappropriate_tool_use_guidelines 208 appropriate_tool_use_guidelines = str( 209 self.eval.template_properties.get("appropriate_tool_use_guidelines") 210 or "" 211 ) 212 inappropriate_tool_use_guidelines = str( 213 self.eval.template_properties.get("inappropriate_tool_use_guidelines") 214 or "" 215 ) 216 217 description += f"""The model was given the following <appropriate_tool_use_guidelines> guidelines: 218<eval_data> 219<appropriate_tool_use_guidelines> 220{appropriate_tool_use_guidelines} 221</appropriate_tool_use_guidelines> 222</eval_data> 223""" 224 # Only include if it has content since it is optional 225 if inappropriate_tool_use_guidelines: 226 description += f"""The model was given the following <inappropriate_tool_use_guidelines> guidelines: 227<eval_data> 228<inappropriate_tool_use_guidelines> 229{inappropriate_tool_use_guidelines} 230</inappropriate_tool_use_guidelines> 231</eval_data> 232""" 233 234 if available_tools is not None: 235 if available_tools != "": 236 description += f""" 237This is the list of tools available to the model: 238<eval_data> 239<available_tools>{available_tools}</available_tools> 240</eval_data> 241""" 242 else: 243 description += """ 244There were no tools available to the model. 245""" 246 247 description += f""" 248This is the full conversation history for the task run: 249<eval_data> 250<conversation_history>{conversation_history}</conversation_history> 251</eval_data> 252""" 253 return description
async def
run_eval( self, task_run: kiln_ai.datamodel.TaskRun, eval_job_item: kiln_ai.datamodel.TaskRun | None = None) -> tuple[typing.Dict[str, float], typing.Optional[typing.Dict[str, str]]]:
255 async def run_eval( 256 self, task_run: TaskRun, eval_job_item: TaskRun | None = None 257 ) -> tuple[EvalScores, Dict[str, str] | None]: 258 """ 259 Run this eval on the given task run. 260 """ 261 262 model_name, provider = self.model_and_provider() 263 264 # Only fetch logprobs for G-Eval 265 # There are at most 5 valid rating tokens per rating type (five_star being largest), so 10 is more than enough to get to the very very unlikely 266 top_logprobs = ( 267 10 if self.eval_config.config_type == EvalConfigType.g_eval else None 268 ) 269 270 # We don't expose setting this manually in the UI, so pull a recommended mode from ml_model_list 271 structured_output_mode = default_structured_output_mode_for_model_provider( 272 model_name, 273 provider, 274 default=StructuredOutputMode.json_schema, 275 # G-eval expects JSON, so don't allow function calling modes 276 disallowed_modes=[ 277 StructuredOutputMode.function_calling, 278 StructuredOutputMode.function_calling_weak, 279 ], 280 ) 281 282 adapter = adapter_for_task( 283 self.geval_task, 284 run_config_properties=KilnAgentRunConfigProperties( 285 model_name=model_name, 286 model_provider_name=provider, 287 # We always use Simple COT for G-Eval and LLM as Judge 288 prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT, 289 structured_output_mode=structured_output_mode, 290 ), 291 base_adapter_config=AdapterConfig( 292 # Don't save this run into the task_runs. It will be saved into an eval_run where it belongs 293 allow_saving=False, 294 top_logprobs=top_logprobs, 295 ), 296 ) 297 298 if self.eval.evaluation_data_type == EvalDataType.full_trace: 299 if task_run.trace is None: 300 raise ValueError("Task run trace is required for full trace evaluation") 301 302 available_tools = await EvalUtils.formatted_available_tools_from_task_run( 303 task_run 304 ) 305 run_description = self.generate_full_trace_run_description( 306 task_run.input, 307 available_tools, 308 EvalTraceFormatter.trace_to_formatted_conversation_history( 309 task_run.trace 310 ), 311 ) 312 313 elif self.eval.evaluation_data_type == EvalDataType.reference_answer: 314 if eval_job_item is None: 315 raise ValueError( 316 "Eval job item is required for reference answer evaluation" 317 ) 318 run_description = self.generate_ref_ans_run_description( 319 task_run.input, task_run.output.output, eval_job_item.output.output 320 ) 321 322 else: # EvalDataType.final_answer 323 run_description = self.generate_final_answer_run_description( 324 task_run.input, task_run.output.output 325 ) 326 327 # We don't need the run, but invoke_returning_run_output() runs validations for us over _run() 328 _, run_output = await adapter.invoke_returning_run_output(run_description) 329 330 if self.eval_config.config_type == EvalConfigType.llm_as_judge: 331 return self.build_llm_as_judge_score( 332 run_output 333 ), run_output.intermediate_outputs 334 else: 335 return self.build_g_eval_score(run_output), run_output.intermediate_outputs
Run this eval on the given task run.
def
build_llm_as_judge_score( self, run_output: kiln_ai.adapters.run_output.RunOutput) -> Dict[str, float]:
337 def build_llm_as_judge_score(self, run_output: RunOutput) -> EvalScores: 338 """Build the LLM as Judge score for the given run and run output.""" 339 from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 340 build_llm_as_judge_score, 341 ) 342 343 return build_llm_as_judge_score(run_output, self.score_from_token_string)
Build the LLM as Judge score for the given run and run output.
def
build_g_eval_score( self, run_output: kiln_ai.adapters.run_output.RunOutput) -> Dict[str, float]:
345 def build_g_eval_score(self, run_output: RunOutput) -> EvalScores: 346 """Build the G-Eval score for the given run and run output. 347 348 We create a weighted average of each rating using the logprobs. 349 350 @misc{liu2023gevalnlgevaluationusing, 351 title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment}, 352 author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu}, 353 year={2023}, 354 eprint={2303.16634}, 355 archivePrefix={arXiv}, 356 primaryClass={cs.CL}, 357 url={https://arxiv.org/abs/2303.16634}, 358 } 359 """ 360 from kiln_ai.adapters.eval.eval_utils.scoring_utils import ( 361 build_g_eval_score, 362 ) 363 364 return build_g_eval_score( 365 run_output, 366 self.raw_output_from_logprobs, 367 self.metric_offsets, 368 self.g_eval_single_metric, 369 )
Build the G-Eval score for the given run and run output.
We create a weighted average of each rating using the logprobs.
@misc{liu2023gevalnlgevaluationusing, title={G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment}, author={Yang Liu and Dan Iter and Yichong Xu and Shuohang Wang and Ruochen Xu and Chenguang Zhu}, year={2023}, eprint={2303.16634}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2303.16634}, }
def
g_eval_single_metric( self, run_output: kiln_ai.adapters.run_output.RunOutput, metric: str, metric_offsets: Dict[str, int], raw_output: str) -> float | None:
def
token_search_range( self, raw_output: str, metric: str, metric_offsets: Dict[str, int]) -> Tuple[int, int]:
def
rating_token_to_score( self, token_logprob: litellm.types.utils.ChatCompletionTokenLogprob) -> float | None: