class TokenizeChatRequest(OpenAIBaseModel):
model: str | None = None
messages: list[ChatCompletionMessageParam]
add_generation_prompt: bool = Field(
default=True,
description=(
"If true, the generation prompt will be added to the chat template. "
"This is a parameter used by chat template in tokenizer config of the "
"model."
),
)
return_token_strs: bool | None = Field(
default=False,
description=(
"If true, also return the token strings corresponding to the token ids."
),
)
continue_final_message: bool = Field(
default=False,
description=(
"If this is set, the chat will be formatted so that the final "
"message in the chat is open-ended, without any EOS tokens. The "
"model will continue this message rather than starting a new one. "
'This allows you to "prefill" part of the model\'s response for it. '
"Cannot be used at the same time as `add_generation_prompt`."
),
)
add_special_tokens: bool = Field(
default=False,
description=(
"If true, special tokens (e.g. BOS) will be added to the prompt "
"on top of what is added by the chat template. "
"For most models, the chat template takes care of adding the "
"special tokens so this should be set to false (as is the "
"default)."
),
)
chat_template: str | None = Field(
default=None,
description=(
"A Jinja template to use for this conversion. "
"As of transformers v4.44, default chat template is no longer "
"allowed, so you must provide a chat template if the tokenizer "
"does not define one."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the template renderer. "
"Will be accessible by the chat template."
),
)
response_format: AnyResponseFormat | None = Field(
default=None,
description="Output format to include in model-specific chat templating.",
)
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
default=None,
description=(
"Additional kwargs to pass to the media IO connectors, "
"keyed by modality. Merged with engine-level media_io_kwargs."
),
)
mm_processor_kwargs: dict[str, Any] | None = Field(
default=None,
description="Additional kwargs to pass to the HF processor.",
)
tools: list[ChatCompletionToolsParam] | None = Field(
default=None,
description="A list of tools the model may call.",
)
@model_validator(mode="before")
@classmethod
def check_generation_prompt(cls, data):
if not isinstance(data, dict):
return data
if data.get("continue_final_message") and data.get("add_generation_prompt"):
raise VLLMValidationError(
"Cannot set both `continue_final_message` and "
"`add_generation_prompt` to True.",
)
return data
def build_chat_params(
self,
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
) -> ChatParams:
return ChatParams(
chat_template=self.chat_template or default_template,
chat_template_content_format=default_template_content_format,
chat_template_kwargs=merge_kwargs(
self.chat_template_kwargs,
dict(
add_generation_prompt=self.add_generation_prompt,
continue_final_message=self.continue_final_message,
),
),
media_io_kwargs=self.media_io_kwargs,
response_format=self.response_format,
)
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return TokenizeParams(
max_total_tokens=None,
max_output_tokens=0,
add_special_tokens=self.add_special_tokens,
)
def to_chat_completion_request(self) -> ChatCompletionRequest:
"""Build the chat completion request with this request's chat fields."""
data = self.model_dump(
include={
"model",
"messages",
"tools",
"add_generation_prompt",
"continue_final_message",
"add_special_tokens",
"chat_template",
"chat_template_kwargs",
"media_io_kwargs",
"mm_processor_kwargs",
"response_format",
},
exclude_none=True,
)
# /tokenize accepts an empty tools list, which the chat route rejects.
if data.get("tools") == []:
del data["tools"]
return ChatCompletionRequest.model_validate(data)