[Bug]: New llama 3.2 vision models from groq not working: prompting with images is incompatible with system messages
Opened this issue · 1 comments
pradhyumna85 commented
What happened?
import litellm
import base64
import imghdr
from importlib.metadata import version
## loading groq api key using .env file
from dotenv import load_dotenv
load_dotenv(override=True)
print("Litellm version: ",version('litellm'))
# Function to encode the image
def encode_image(image_path):
with open(image_path, "rb") as image_file:
return imghdr.what(image_path),base64.b64encode(image_file.read()).decode('utf-8')
image_path = "test_image.jpg"
# Getting the base64 string
image_type, base64_image = encode_image(image_path)
model = "groq/llama-3.2-11b-vision-preview"
messages=[
{
"role": "system",
"content": "extract the text as markdown, and don't miss any details. Only respond with the markdown without root ``` tag without any explanation"
},
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:image/{image_type};base64,{base64_image}",
},
},
],
}
]
response = litellm.completion(model=model,messages=messages)
print(response.choices[0].message.content)I think this needs to be handled in the litellm backend.
related: getomni-ai/zerox#65
Relevant log output
---------------------------------------------------------------------------
BadRequestError Traceback (most recent call last)
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\llms\OpenAI\openai.py:854, in OpenAIChatCompletion.completion(self, model_response, timeout, optional_params, logging_obj, model, messages, print_verbose, api_key, api_base, acompletion, litellm_params, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params)
853 else:
--> 854 raise e
855 except OpenAIError as e:
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\llms\OpenAI\openai.py:790, in OpenAIChatCompletion.completion(self, model_response, timeout, optional_params, logging_obj, model, messages, print_verbose, api_key, api_base, acompletion, litellm_params, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params)
778 logging_obj.pre_call(
779 input=messages,
780 api_key=openai_client.api_key,
(...)
786 },
787 )
789 headers, response = (
--> 790 self.make_sync_openai_chat_completion_request(
791 openai_client=openai_client,
792 data=data,
793 timeout=timeout,
794 )
795 )
797 logging_obj.model_call_details["response_headers"] = headers
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\llms\OpenAI\openai.py:649, in OpenAIChatCompletion.make_sync_openai_chat_completion_request(self, openai_client, data, timeout)
648 else:
--> 649 raise e
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\llms\OpenAI\openai.py:631, in OpenAIChatCompletion.make_sync_openai_chat_completion_request(self, openai_client, data, timeout)
630 try:
--> 631 raw_response = openai_client.chat.completions.with_raw_response.create(
632 **data, timeout=timeout
633 )
635 if hasattr(raw_response, "headers"):
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\openai\_legacy_response.py:356, in to_raw_response_wrapper.<locals>.wrapped(*args, **kwargs)
354 kwargs["extra_headers"] = extra_headers
--> 356 return cast(LegacyAPIResponse[R], func(*args, **kwargs))
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\openai\_utils\_utils.py:274, in required_args.<locals>.inner.<locals>.wrapper(*args, **kwargs)
273 raise TypeError(msg)
--> 274 return func(*args, **kwargs)
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\openai\resources\chat\completions.py:815, in Completions.create(self, messages, model, audio, frequency_penalty, function_call, functions, logit_bias, logprobs, max_completion_tokens, max_tokens, metadata, modalities, n, parallel_tool_calls, presence_penalty, response_format, seed, service_tier, stop, store, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)
814 validate_response_format(response_format)
--> 815 return self._post(
816 "/chat/completions",
817 body=maybe_transform(
818 {
819 "messages": messages,
820 "model": model,
821 "audio": audio,
822 "frequency_penalty": frequency_penalty,
823 "function_call": function_call,
824 "functions": functions,
825 "logit_bias": logit_bias,
826 "logprobs": logprobs,
827 "max_completion_tokens": max_completion_tokens,
828 "max_tokens": max_tokens,
829 "metadata": metadata,
830 "modalities": modalities,
831 "n": n,
832 "parallel_tool_calls": parallel_tool_calls,
833 "presence_penalty": presence_penalty,
834 "response_format": response_format,
835 "seed": seed,
836 "service_tier": service_tier,
837 "stop": stop,
838 "store": store,
839 "stream": stream,
840 "stream_options": stream_options,
841 "temperature": temperature,
842 "tool_choice": tool_choice,
843 "tools": tools,
844 "top_logprobs": top_logprobs,
845 "top_p": top_p,
846 "user": user,
847 },
848 completion_create_params.CompletionCreateParams,
849 ),
850 options=make_request_options(
851 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
852 ),
853 cast_to=ChatCompletion,
854 stream=stream or False,
855 stream_cls=Stream[ChatCompletionChunk],
856 )
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\openai\_base_client.py:1277, in SyncAPIClient.post(self, path, cast_to, body, options, files, stream, stream_cls)
1274 opts = FinalRequestOptions.construct(
1275 method="post", url=path, json_data=body, files=to_httpx_files(files), **options
1276 )
-> 1277 return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\openai\_base_client.py:954, in SyncAPIClient.request(self, cast_to, options, remaining_retries, stream, stream_cls)
952 retries_taken = 0
--> 954 return self._request(
955 cast_to=cast_to,
956 options=options,
957 stream=stream,
958 stream_cls=stream_cls,
959 retries_taken=retries_taken,
960 )
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\openai\_base_client.py:1058, in SyncAPIClient._request(self, cast_to, options, retries_taken, stream, stream_cls)
1057 log.debug("Re-raising status error")
-> 1058 raise self._make_status_error_from_response(err.response) from None
1060 return self._process_response(
1061 cast_to=cast_to,
1062 options=options,
(...)
1066 retries_taken=retries_taken,
1067 )
BadRequestError: Error code: 400 - {'error': {'message': 'prompting with images is incompatible with system messages', 'type': 'invalid_request_error'}}
During handling of the above exception, another exception occurred:
OpenAIError Traceback (most recent call last)
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\main.py:1463, in completion(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, audio, presence_penalty, frequency_penalty, logit_bias, user, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, deployment_id, extra_headers, functions, function_call, base_url, api_version, api_key, model_list, **kwargs)
1461 optional_params[k] = v
-> 1463 response = groq_chat_completions.completion(
1464 model=model,
1465 messages=messages,
1466 headers=headers,
1467 model_response=model_response,
1468 print_verbose=print_verbose,
1469 api_key=api_key,
1470 api_base=api_base,
1471 acompletion=acompletion,
1472 logging_obj=logging,
1473 optional_params=optional_params,
1474 litellm_params=litellm_params,
1475 logger_fn=logger_fn,
1476 timeout=timeout, # type: ignore
1477 custom_prompt_dict=custom_prompt_dict,
1478 client=client, # pass AsyncOpenAI, OpenAI client
1479 organization=organization,
1480 custom_llm_provider=custom_llm_provider,
1481 )
1482 elif (
1483 model in litellm.open_ai_chat_completion_models
1484 or custom_llm_provider == "custom_openai"
(...)
1501 # note: if a user sets a custom base - we should ensure this works
1502 # allow for the setting of dynamic and stateful api-bases
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\llms\groq\chat\handler.py:41, in GroqChatCompletion.completion(self, model_response, timeout, optional_params, logging_obj, model, messages, print_verbose, api_key, api_base, acompletion, litellm_params, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params)
40 messages = GroqChatConfig()._transform_messages(messages) # type: ignore
---> 41 return super().completion(
42 model_response,
43 timeout,
44 optional_params,
45 logging_obj,
46 model,
47 messages,
48 print_verbose,
49 api_key,
50 api_base,
51 acompletion,
52 litellm_params,
53 logger_fn,
54 headers,
55 custom_prompt_dict,
56 client,
57 organization,
58 custom_llm_provider,
59 drop_params,
60 )
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\llms\OpenAI\openai.py:864, in OpenAIChatCompletion.completion(self, model_response, timeout, optional_params, logging_obj, model, messages, print_verbose, api_key, api_base, acompletion, litellm_params, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params)
863 error_headers = getattr(error_response, "headers", None)
--> 864 raise OpenAIError(
865 status_code=status_code, message=error_text, headers=error_headers
866 )
OpenAIError: Error code: 400 - {'error': {'message': 'prompting with images is incompatible with system messages', 'type': 'invalid_request_error'}}
During handling of the above exception, another exception occurred:
BadRequestError Traceback (most recent call last)
Cell In[20], line 42
20 model = "groq/llama-3.2-11b-vision-preview"
22 messages=[
23 {
24 "role": "system",
(...)
38 }
39 ]
---> 42 response = litellm.completion(model=model,messages=messages)
44 print(response.choices[0].message.content)
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\utils.py:1013, in client.<locals>.wrapper(*args, **kwargs)
1009 if logging_obj:
1010 logging_obj.failure_handler(
1011 e, traceback_exception, start_time, end_time
1012 ) # DO NOT MAKE THREADED - router retry fallback relies on this!
-> 1013 raise e
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\utils.py:903, in client.<locals>.wrapper(*args, **kwargs)
901 print_verbose(f"Error while checking max token limit: {str(e)}")
902 # MODEL CALL
--> 903 result = original_function(*args, **kwargs)
904 end_time = datetime.datetime.now()
905 if "stream" in kwargs and kwargs["stream"] is True:
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\main.py:2998, in completion(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, audio, presence_penalty, frequency_penalty, logit_bias, user, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, deployment_id, extra_headers, functions, function_call, base_url, api_version, api_key, model_list, **kwargs)
2995 return response
2996 except Exception as e:
2997 ## Map to OpenAI Exception
-> 2998 raise exception_type(
2999 model=model,
3000 custom_llm_provider=custom_llm_provider,
3001 original_exception=e,
3002 completion_kwargs=args,
3003 extra_kwargs=kwargs,
3004 )
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\litellm_core_utils\exception_mapping_utils.py:2116, in exception_type(model, original_exception, custom_llm_provider, completion_kwargs, extra_kwargs)
2114 if exception_mapping_worked:
2115 setattr(e, "litellm_response_headers", litellm_response_headers)
-> 2116 raise e
2117 else:
2118 for error_type in litellm.LITELLM_EXCEPTION_TYPES:
File d:\installed_softwares\miniforge3\envs\zeroxocr\Lib\site-packages\litellm\litellm_core_utils\exception_mapping_utils.py:282, in exception_type(model, original_exception, custom_llm_provider, completion_kwargs, extra_kwargs)
277 elif (
278 "invalid_request_error" in error_str
279 and "Incorrect API key provided" not in error_str
280 ):
281 exception_mapping_worked = True
--> 282 raise BadRequestError(
283 message=f"{exception_provider} - {message}",
284 llm_provider=custom_llm_provider,
285 model=model,
286 response=original_exception.response,
287 litellm_debug_info=extra_information,
288 )
289 elif "Web server is returning an unknown error" in error_str:
290 exception_mapping_worked = True
BadRequestError: litellm.BadRequestError: GroqException - Error code: 400 - {'error': {'message': 'prompting with images is incompatible with system messages', 'type': 'invalid_request_error'}}Twitter / LinkedIn details
newptcai commented
I have contacted Groq. They told me at the moment system prompt and image cannot be in the same message.