kiln_ai.adapters.model_adapters.litellm_adapter

  1import copy
  2import json
  3import logging
  4from dataclasses import dataclass
  5from typing import Any, Dict, List, Tuple, TypeAlias, Union
  6
  7import litellm
  8from litellm.types.utils import (
  9    ChatCompletionMessageToolCall,
 10    ChoiceLogprobs,
 11    Choices,
 12    ModelResponse,
 13)
 14from litellm.types.utils import Message as LiteLLMMessage
 15from litellm.types.utils import Usage as LiteLlmUsage
 16from openai.types.chat.chat_completion_message_tool_call_param import (
 17    ChatCompletionMessageToolCallParam,
 18)
 19
 20import kiln_ai.datamodel as datamodel
 21from kiln_ai.adapters.ml_model_list import (
 22    KilnModelProvider,
 23    ModelProviderName,
 24    StructuredOutputMode,
 25)
 26from kiln_ai.adapters.model_adapters.base_adapter import (
 27    AdapterConfig,
 28    BaseAdapter,
 29    RunOutput,
 30    Usage,
 31)
 32from kiln_ai.adapters.model_adapters.litellm_config import LiteLlmConfig
 33from kiln_ai.datamodel.datamodel_enums import InputType
 34from kiln_ai.datamodel.json_schema import validate_schema_with_value_error
 35from kiln_ai.tools.base_tool import (
 36    KilnToolInterface,
 37    ToolCallContext,
 38    ToolCallDefinition,
 39)
 40from kiln_ai.tools.kiln_task_tool import KilnTaskToolResult
 41from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 42from kiln_ai.utils.litellm import get_litellm_provider_info
 43from kiln_ai.utils.open_ai_types import (
 44    ChatCompletionAssistantMessageParamWrapper,
 45    ChatCompletionMessageParam,
 46    ChatCompletionToolMessageParamWrapper,
 47)
 48
 49MAX_CALLS_PER_TURN = 10
 50MAX_TOOL_CALLS_PER_TURN = 30
 51
 52logger = logging.getLogger(__name__)
 53
 54ChatCompletionMessageIncludingLiteLLM: TypeAlias = Union[
 55    ChatCompletionMessageParam, LiteLLMMessage
 56]
 57
 58
 59@dataclass
 60class ModelTurnResult:
 61    assistant_message: str
 62    all_messages: list[ChatCompletionMessageIncludingLiteLLM]
 63    model_response: ModelResponse | None
 64    model_choice: Choices | None
 65    usage: Usage
 66
 67
 68class LiteLlmAdapter(BaseAdapter):
 69    def __init__(
 70        self,
 71        config: LiteLlmConfig,
 72        kiln_task: datamodel.Task,
 73        base_adapter_config: AdapterConfig | None = None,
 74    ):
 75        self.config = config
 76        self._additional_body_options = config.additional_body_options
 77        self._api_base = config.base_url
 78        self._headers = config.default_headers
 79        self._litellm_model_id: str | None = None
 80        self._cached_available_tools: list[KilnToolInterface] | None = None
 81
 82        super().__init__(
 83            task=kiln_task,
 84            run_config=config.run_config_properties,
 85            config=base_adapter_config,
 86        )
 87
 88    async def _run_model_turn(
 89        self,
 90        provider: KilnModelProvider,
 91        prior_messages: list[ChatCompletionMessageIncludingLiteLLM],
 92        top_logprobs: int | None,
 93        skip_response_format: bool,
 94    ) -> ModelTurnResult:
 95        """
 96        Call the model for a single top level turn: from user message to agent message.
 97
 98        It may make handle iterations of tool calls between the user/agent message if needed.
 99        """
100
101        usage = Usage()
102        messages = list(prior_messages)
103        tool_calls_count = 0
104
105        while tool_calls_count < MAX_TOOL_CALLS_PER_TURN:
106            # Build completion kwargs for tool calls
107            completion_kwargs = await self.build_completion_kwargs(
108                provider,
109                # Pass a copy, as acompletion mutates objects and breaks types.
110                copy.deepcopy(messages),
111                top_logprobs,
112                skip_response_format,
113            )
114
115            # Make the completion call
116            model_response, response_choice = await self.acompletion_checking_response(
117                **completion_kwargs
118            )
119
120            # count the usage
121            usage += self.usage_from_response(model_response)
122
123            # Extract content and tool calls
124            if not hasattr(response_choice, "message"):
125                raise ValueError("Response choice has no message")
126            content = response_choice.message.content
127            tool_calls = response_choice.message.tool_calls
128            if not content and not tool_calls:
129                raise ValueError(
130                    "Model returned an assistant message, but no content or tool calls. This is not supported."
131                )
132
133            # Add message to messages, so it can be used in the next turn
134            messages.append(response_choice.message)
135
136            # Process tool calls if any
137            if tool_calls and len(tool_calls) > 0:
138                (
139                    assistant_message_from_toolcall,
140                    tool_call_messages,
141                ) = await self.process_tool_calls(tool_calls)
142
143                # Add tool call results to messages
144                messages.extend(tool_call_messages)
145
146                # If task_response tool was called, we're done
147                if assistant_message_from_toolcall is not None:
148                    return ModelTurnResult(
149                        assistant_message=assistant_message_from_toolcall,
150                        all_messages=messages,
151                        model_response=model_response,
152                        model_choice=response_choice,
153                        usage=usage,
154                    )
155
156                # If there were tool calls, increment counter and continue
157                if tool_call_messages:
158                    tool_calls_count += 1
159                    continue
160
161            # If no tool calls, return the content as final output
162            if content:
163                return ModelTurnResult(
164                    assistant_message=content,
165                    all_messages=messages,
166                    model_response=model_response,
167                    model_choice=response_choice,
168                    usage=usage,
169                )
170
171            # If we get here with no content and no tool calls, break
172            raise RuntimeError(
173                "Model returned neither content nor tool calls. It must return at least one of these."
174            )
175
176        raise RuntimeError(
177            f"Too many tool calls ({tool_calls_count}). Stopping iteration to avoid using too many tokens."
178        )
179
180    async def _run(self, input: InputType) -> tuple[RunOutput, Usage | None]:
181        usage = Usage()
182
183        provider = self.model_provider()
184        if not provider.model_id:
185            raise ValueError("Model ID is required for OpenAI compatible models")
186
187        chat_formatter = self.build_chat_formatter(input)
188        messages: list[ChatCompletionMessageIncludingLiteLLM] = []
189
190        prior_output: str | None = None
191        final_choice: Choices | None = None
192        turns = 0
193
194        while True:
195            turns += 1
196            if turns > MAX_CALLS_PER_TURN:
197                raise RuntimeError(
198                    f"Too many turns ({turns}). Stopping iteration to avoid using too many tokens."
199                )
200
201            turn = chat_formatter.next_turn(prior_output)
202            if turn is None:
203                # No next turn, we're done
204                break
205
206            # Add messages from the turn to chat history
207            for message in turn.messages:
208                if message.content is None:
209                    raise ValueError("Empty message content isn't allowed")
210                # pyright incorrectly warns about this, but it's valid so we can ignore. It can't handle the multi-value role.
211                messages.append({"role": message.role, "content": message.content})  # type: ignore
212
213            skip_response_format = not turn.final_call
214            turn_result = await self._run_model_turn(
215                provider,
216                messages,
217                self.base_adapter_config.top_logprobs if turn.final_call else None,
218                skip_response_format,
219            )
220
221            usage += turn_result.usage
222
223            prior_output = turn_result.assistant_message
224            messages = turn_result.all_messages
225            final_choice = turn_result.model_choice
226
227            if not prior_output:
228                raise RuntimeError("No assistant message/output returned from model")
229
230        logprobs = self._extract_and_validate_logprobs(final_choice)
231
232        # Save COT/reasoning if it exists. May be a message, or may be parsed by LiteLLM (or openrouter, or anyone upstream)
233        intermediate_outputs = chat_formatter.intermediate_outputs()
234        self._extract_reasoning_to_intermediate_outputs(
235            final_choice, intermediate_outputs
236        )
237
238        if not isinstance(prior_output, str):
239            raise RuntimeError(f"assistant message is not a string: {prior_output}")
240
241        trace = self.all_messages_to_trace(messages)
242        output = RunOutput(
243            output=prior_output,
244            intermediate_outputs=intermediate_outputs,
245            output_logprobs=logprobs,
246            trace=trace,
247        )
248
249        return output, usage
250
251    def _extract_and_validate_logprobs(
252        self, final_choice: Choices | None
253    ) -> ChoiceLogprobs | None:
254        """
255        Extract logprobs from the final choice and validate they exist if required.
256        """
257        logprobs = None
258        if (
259            final_choice is not None
260            and hasattr(final_choice, "logprobs")
261            and isinstance(final_choice.logprobs, ChoiceLogprobs)
262        ):
263            logprobs = final_choice.logprobs
264
265        # Check logprobs worked, if required
266        if self.base_adapter_config.top_logprobs is not None and logprobs is None:
267            raise RuntimeError("Logprobs were required, but no logprobs were returned.")
268
269        return logprobs
270
271    def _extract_reasoning_to_intermediate_outputs(
272        self, final_choice: Choices | None, intermediate_outputs: Dict[str, Any]
273    ) -> None:
274        """Extract reasoning content from model choice and add to intermediate outputs if present."""
275        if (
276            final_choice is not None
277            and hasattr(final_choice, "message")
278            and hasattr(final_choice.message, "reasoning_content")
279        ):
280            reasoning_content = final_choice.message.reasoning_content
281            if reasoning_content is not None:
282                stripped_reasoning_content = reasoning_content.strip()
283                if len(stripped_reasoning_content) > 0:
284                    intermediate_outputs["reasoning"] = stripped_reasoning_content
285
286    async def acompletion_checking_response(
287        self, **kwargs
288    ) -> Tuple[ModelResponse, Choices]:
289        response = await litellm.acompletion(**kwargs)
290        if (
291            not isinstance(response, ModelResponse)
292            or not response.choices
293            or len(response.choices) == 0
294            or not isinstance(response.choices[0], Choices)
295        ):
296            raise RuntimeError(
297                f"Expected ModelResponse with Choices, got {type(response)}."
298            )
299        return response, response.choices[0]
300
301    def adapter_name(self) -> str:
302        return "kiln_openai_compatible_adapter"
303
304    async def response_format_options(self) -> dict[str, Any]:
305        # Unstructured if task isn't structured
306        if not self.has_structured_output():
307            return {}
308
309        structured_output_mode = self.run_config.structured_output_mode
310
311        match structured_output_mode:
312            case StructuredOutputMode.json_mode:
313                return {"response_format": {"type": "json_object"}}
314            case StructuredOutputMode.json_schema:
315                return self.json_schema_response_format()
316            case StructuredOutputMode.function_calling_weak:
317                return self.tool_call_params(strict=False)
318            case StructuredOutputMode.function_calling:
319                return self.tool_call_params(strict=True)
320            case StructuredOutputMode.json_instructions:
321                # JSON instructions dynamically injected in prompt, not the API response format. Do not ask for json_object (see option below).
322                return {}
323            case StructuredOutputMode.json_custom_instructions:
324                # JSON instructions statically injected in system prompt, not the API response format. Do not ask for json_object (see option above).
325                return {}
326            case StructuredOutputMode.json_instruction_and_object:
327                # We set response_format to json_object and also set json instructions in the prompt
328                return {"response_format": {"type": "json_object"}}
329            case StructuredOutputMode.default:
330                provider_name = self.run_config.model_provider_name
331                if provider_name == ModelProviderName.ollama:
332                    # Ollama added json_schema to all models: https://ollama.com/blog/structured-outputs
333                    return self.json_schema_response_format()
334                elif provider_name == ModelProviderName.docker_model_runner:
335                    # Docker Model Runner uses OpenAI-compatible API with JSON schema support
336                    return self.json_schema_response_format()
337                else:
338                    # Default to function calling -- it's older than the other modes. Higher compatibility.
339                    # Strict isn't widely supported yet, so we don't use it by default unless it's OpenAI.
340                    strict = provider_name == ModelProviderName.openai
341                    return self.tool_call_params(strict=strict)
342            case StructuredOutputMode.unknown:
343                # See above, but this case should never happen.
344                raise ValueError("Structured output mode is unknown.")
345            case _:
346                raise_exhaustive_enum_error(structured_output_mode)
347
348    def json_schema_response_format(self) -> dict[str, Any]:
349        output_schema = self.task.output_schema()
350        return {
351            "response_format": {
352                "type": "json_schema",
353                "json_schema": {
354                    "name": "task_response",
355                    "schema": output_schema,
356                },
357            }
358        }
359
360    def tool_call_params(self, strict: bool) -> dict[str, Any]:
361        # Add additional_properties: false to the schema (OpenAI requires this for some models)
362        output_schema = self.task.output_schema()
363        if not isinstance(output_schema, dict):
364            raise ValueError(
365                "Invalid output schema for this task. Can not use tool calls."
366            )
367        output_schema["additionalProperties"] = False
368
369        function_params = {
370            "name": "task_response",
371            "parameters": output_schema,
372        }
373        # This should be on, but we allow setting function_calling_weak for APIs that don't support it.
374        if strict:
375            function_params["strict"] = True
376
377        return {
378            "tools": [
379                {
380                    "type": "function",
381                    "function": function_params,
382                }
383            ],
384            "tool_choice": {
385                "type": "function",
386                "function": {"name": "task_response"},
387            },
388        }
389
390    def build_extra_body(self, provider: KilnModelProvider) -> dict[str, Any]:
391        # Don't love having this logic here. But it's worth the usability improvement
392        # so better to keep it than exclude it. Should figure out how I want to isolate
393        # this sort of logic so it's config driven and can be overridden
394
395        extra_body = {}
396        provider_options = {}
397
398        if provider.thinking_level is not None:
399            extra_body["reasoning_effort"] = provider.thinking_level
400
401        if provider.require_openrouter_reasoning:
402            # https://openrouter.ai/docs/use-cases/reasoning-tokens
403            extra_body["reasoning"] = {
404                "exclude": False,
405            }
406
407        if provider.gemini_reasoning_enabled:
408            extra_body["reasoning"] = {
409                "enabled": True,
410            }
411
412        if provider.name == ModelProviderName.openrouter:
413            # Ask OpenRouter to include usage in the response (cost)
414            extra_body["usage"] = {"include": True}
415
416            # Set a default provider order for more deterministic routing.
417            # OpenRouter will ignore providers that don't support the model.
418            # Special cases below (like R1) can override this order.
419            # allow_fallbacks is true by default, but we can override it here.
420            provider_options["order"] = [
421                "fireworks",
422                "parasail",
423                "together",
424                "deepinfra",
425                "novita",
426                "groq",
427                "amazon-bedrock",
428                "azure",
429                "nebius",
430            ]
431
432        if provider.anthropic_extended_thinking:
433            extra_body["thinking"] = {"type": "enabled", "budget_tokens": 4000}
434
435        if provider.r1_openrouter_options:
436            # Require providers that support the reasoning parameter
437            provider_options["require_parameters"] = True
438            # Prefer R1 providers with reasonable perf/quants
439            provider_options["order"] = ["fireworks", "together"]
440            # R1 providers with unreasonable quants
441            provider_options["ignore"] = ["deepinfra"]
442
443        # Only set of this request is to get logprobs.
444        if (
445            provider.logprobs_openrouter_options
446            and self.base_adapter_config.top_logprobs is not None
447        ):
448            # Don't let OpenRouter choose a provider that doesn't support logprobs.
449            provider_options["require_parameters"] = True
450            # DeepInfra silently fails to return logprobs consistently.
451            provider_options["ignore"] = ["deepinfra"]
452
453        if provider.openrouter_skip_required_parameters:
454            # Oddball case, R1 14/8/1.5B fail with this param, even though they support thinking params.
455            provider_options["require_parameters"] = False
456
457        # Siliconflow uses a bool flag for thinking, for some models
458        if provider.siliconflow_enable_thinking is not None:
459            extra_body["enable_thinking"] = provider.siliconflow_enable_thinking
460
461        if len(provider_options) > 0:
462            extra_body["provider"] = provider_options
463
464        return extra_body
465
466    def litellm_model_id(self) -> str:
467        # The model ID is an interesting combination of format and url endpoint.
468        # It specifics the provider URL/host, but this is overridden if you manually set an api url
469        if self._litellm_model_id:
470            return self._litellm_model_id
471
472        litellm_provider_info = get_litellm_provider_info(self.model_provider())
473        if litellm_provider_info.is_custom and self._api_base is None:
474            raise ValueError(
475                "Explicit Base URL is required for OpenAI compatible APIs (custom models, ollama, fine tunes, and custom registry models)"
476            )
477
478        self._litellm_model_id = litellm_provider_info.litellm_model_id
479        return self._litellm_model_id
480
481    async def build_completion_kwargs(
482        self,
483        provider: KilnModelProvider,
484        messages: list[ChatCompletionMessageIncludingLiteLLM],
485        top_logprobs: int | None,
486        skip_response_format: bool = False,
487    ) -> dict[str, Any]:
488        extra_body = self.build_extra_body(provider)
489
490        # Merge all parameters into a single kwargs dict for litellm
491        completion_kwargs = {
492            "model": self.litellm_model_id(),
493            "messages": messages,
494            "api_base": self._api_base,
495            "headers": self._headers,
496            "temperature": self.run_config.temperature,
497            "top_p": self.run_config.top_p,
498            # This drops params that are not supported by the model. Only openai params like top_p, temperature -- not litellm params like model, etc.
499            # Not all models and providers support all openai params (for example, o3 doesn't support top_p)
500            # Better to ignore them than to fail the model call.
501            # https://docs.litellm.ai/docs/completion/input
502            "drop_params": True,
503            **extra_body,
504            **self._additional_body_options,
505        }
506
507        tool_calls = await self.litellm_tools()
508        has_tools = len(tool_calls) > 0
509        if has_tools:
510            completion_kwargs["tools"] = tool_calls
511            completion_kwargs["tool_choice"] = "auto"
512
513        # Special condition for Claude Opus 4.1 and Sonnet 4.5, where we can only specify top_p or temp, not both.
514        # Remove default values (1.0) prioritizing anything the user customized, then error with helpful message if they are both custom.
515        if provider.temp_top_p_exclusive:
516            if "top_p" in completion_kwargs and completion_kwargs["top_p"] == 1.0:
517                del completion_kwargs["top_p"]
518            if (
519                "temperature" in completion_kwargs
520                and completion_kwargs["temperature"] == 1.0
521            ):
522                del completion_kwargs["temperature"]
523            if "top_p" in completion_kwargs and "temperature" in completion_kwargs:
524                raise ValueError(
525                    "top_p and temperature can not both have custom values for this model. This is a restriction from the model provider. Please set only one of them to a custom value (not 1.0)."
526                )
527
528        if not skip_response_format:
529            # Response format: json_schema, json_instructions, json_mode, function_calling, etc
530            response_format_options = await self.response_format_options()
531
532            # Check for a conflict between tools and response format using tools
533            # We could reconsider this. Model could be able to choose between a final answer or a tool call on any turn. However, good models for tools tend to also support json_schea, so do we need to support both? If we do, merge them, and consider auto vs forced when merging (only forced for final, auto for merged).
534            if has_tools and "tools" in response_format_options:
535                raise ValueError(
536                    "Function calling/tools can't be used as the JSON response format if you're also using tools. Please select a different structured output mode."
537                )
538
539            completion_kwargs.update(response_format_options)
540
541        if top_logprobs is not None:
542            completion_kwargs["logprobs"] = True
543            completion_kwargs["top_logprobs"] = top_logprobs
544
545        return completion_kwargs
546
547    def usage_from_response(self, response: ModelResponse) -> Usage:
548        litellm_usage = response.get("usage", None)
549
550        # LiteLLM isn't consistent in how it returns the cost.
551        cost = response._hidden_params.get("response_cost", None)
552        if cost is None and litellm_usage:
553            cost = litellm_usage.get("cost", None)
554
555        usage = Usage()
556
557        if not litellm_usage and not cost:
558            return usage
559
560        if litellm_usage and isinstance(litellm_usage, LiteLlmUsage):
561            usage.input_tokens = litellm_usage.get("prompt_tokens", None)
562            usage.output_tokens = litellm_usage.get("completion_tokens", None)
563            usage.total_tokens = litellm_usage.get("total_tokens", None)
564        else:
565            logger.warning(
566                f"Unexpected usage format from litellm: {litellm_usage}. Expected Usage object, got {type(litellm_usage)}"
567            )
568
569        if isinstance(cost, float):
570            usage.cost = cost
571        elif cost is not None:
572            # None is allowed, but no other types are expected
573            logger.warning(
574                f"Unexpected cost format from litellm: {cost}. Expected float, got {type(cost)}"
575            )
576
577        return usage
578
579    async def cached_available_tools(self) -> list[KilnToolInterface]:
580        if self._cached_available_tools is None:
581            self._cached_available_tools = await self.available_tools()
582        return self._cached_available_tools
583
584    async def litellm_tools(self) -> list[ToolCallDefinition]:
585        available_tools = await self.cached_available_tools()
586
587        # LiteLLM takes the standard OpenAI-compatible tool call format
588        return [await tool.toolcall_definition() for tool in available_tools]
589
590    async def process_tool_calls(
591        self, tool_calls: list[ChatCompletionMessageToolCall] | None
592    ) -> tuple[str | None, list[ChatCompletionToolMessageParamWrapper]]:
593        if tool_calls is None:
594            return None, []
595
596        assistant_output_from_toolcall: str | None = None
597        tool_call_response_messages: list[ChatCompletionToolMessageParamWrapper] = []
598
599        for tool_call in tool_calls:
600            # Kiln "task_response" tool is used for returning structured output via tool calls.
601            # Load the output from the tool call. Also
602            if tool_call.function.name == "task_response":
603                assistant_output_from_toolcall = tool_call.function.arguments
604                continue
605
606            # Process normal tool calls (not the "task_response" tool)
607            tool_name = tool_call.function.name
608            tool = None
609            for tool_option in await self.cached_available_tools():
610                if await tool_option.name() == tool_name:
611                    tool = tool_option
612                    break
613            if not tool:
614                raise RuntimeError(
615                    f"A tool named '{tool_name}' was invoked by a model, but was not available."
616                )
617
618            # Parse the arguments and validate them against the tool's schema
619            try:
620                parsed_args = json.loads(tool_call.function.arguments)
621            except json.JSONDecodeError:
622                raise RuntimeError(
623                    f"Failed to parse arguments for tool '{tool_name}' (should be JSON): {tool_call.function.arguments}"
624                )
625            try:
626                tool_call_definition = await tool.toolcall_definition()
627                json_schema = json.dumps(tool_call_definition["function"]["parameters"])
628                validate_schema_with_value_error(parsed_args, json_schema)
629            except Exception as e:
630                raise RuntimeError(
631                    f"Failed to validate arguments for tool '{tool_name}'. The arguments didn't match the tool's schema. The arguments were: {parsed_args}\n The error was: {e}"
632                ) from e
633
634            # Create context with the calling task's allow_saving setting
635            context = ToolCallContext(
636                allow_saving=self.base_adapter_config.allow_saving
637            )
638            result = await tool.run(context, **parsed_args)
639
640            tool_call_response_messages.append(
641                ChatCompletionToolMessageParamWrapper(
642                    role="tool",
643                    tool_call_id=tool_call.id,
644                    content=result.output,
645                    kiln_task_tool_data=result.kiln_task_tool_data
646                    if isinstance(result, KilnTaskToolResult)
647                    else None,
648                )
649            )
650
651        if (
652            assistant_output_from_toolcall is not None
653            and len(tool_call_response_messages) > 0
654        ):
655            raise RuntimeError(
656                "Model asked for impossible combination: task_response tool call and other tool calls were both provided in the same turn. This is not supported as it means the model asked us to both return task_response results (ending the turn) and run new tools calls to send back to the model. If the model makes this mistake often, try a difference structured data model like JSON schema, where this is impossible."
657            )
658
659        return assistant_output_from_toolcall, tool_call_response_messages
660
661    def litellm_message_to_trace_message(
662        self, raw_message: LiteLLMMessage
663    ) -> ChatCompletionAssistantMessageParamWrapper:
664        """
665        Convert a LiteLLM Message object to an OpenAI compatible message, our ChatCompletionAssistantMessageParamWrapper
666        """
667        message: ChatCompletionAssistantMessageParamWrapper = {
668            "role": "assistant",
669        }
670        if raw_message.role != "assistant":
671            raise ValueError(
672                "Model returned a message with a role other than assistant. This is not supported."
673            )
674
675        if hasattr(raw_message, "content"):
676            message["content"] = raw_message.content
677        if hasattr(raw_message, "reasoning_content"):
678            message["reasoning_content"] = raw_message.reasoning_content
679        if hasattr(raw_message, "tool_calls"):
680            # Convert ChatCompletionMessageToolCall to ChatCompletionMessageToolCallParam
681            open_ai_tool_calls: List[ChatCompletionMessageToolCallParam] = []
682            for litellm_tool_call in raw_message.tool_calls or []:
683                # Optional in the SDK for streaming responses, but should never be None at this point.
684                if litellm_tool_call.function.name is None:
685                    raise ValueError(
686                        "The model requested a tool call, without providing a function name (required)."
687                    )
688                open_ai_tool_calls.append(
689                    ChatCompletionMessageToolCallParam(
690                        id=litellm_tool_call.id,
691                        type="function",
692                        function={
693                            "name": litellm_tool_call.function.name,
694                            "arguments": litellm_tool_call.function.arguments,
695                        },
696                    )
697                )
698            if len(open_ai_tool_calls) > 0:
699                message["tool_calls"] = open_ai_tool_calls
700
701        if not message.get("content") and not message.get("tool_calls"):
702            raise ValueError(
703                "Model returned an assistant message, but no content or tool calls. This is not supported."
704            )
705
706        return message
707
708    def all_messages_to_trace(
709        self, messages: list[ChatCompletionMessageIncludingLiteLLM]
710    ) -> list[ChatCompletionMessageParam]:
711        """
712        Internally we allow LiteLLM Message objects, but for trace we need OpenAI compatible types. Replace LiteLLM Message objects with OpenAI compatible types.
713        """
714        trace: list[ChatCompletionMessageParam] = []
715        for message in messages:
716            if isinstance(message, LiteLLMMessage):
717                trace.append(self.litellm_message_to_trace_message(message))
718            else:
719                trace.append(message)
720        return trace