Replicate Streamlit Langchain Streaming + Vector DB Assertion Error

[OlexiyPukhov]

class StreamHandler(BaseCallbackHandler):
    def __init__(self, container, initial_text=""):
        self.container = container
        self.text = initial_text

    def on_llm_new_token(self, token: str, **kwargs) -> None:
        self.text += token
        self.container.markdown(self.text)

docs = doc_store.similarity_search( # vector db
                        conversation_history + user_input, k=3, consistency="all")
                    
                    conversation_history = "\n".join([f"{msg.type}: {msg.content}" for msg in st.session_state.messages[-11:-1:2]])
                    
                    supabase.table('users') \ #database items
                    .update({'query_count': query_count_user - 1}) \
                    .eq('id', user_id) \
                    .execute() 

                    reduce_template = """
Some prompt here.
                    Answer:"""
                    stream_handler = StreamHandler(st.empty())
                    prompt = PromptTemplate.from_template(reduce_template)
                    llm = Replicate(
                        streaming=True,
                        callbacks=[stream_handler],
                        model="a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5",
                        model_kwargs={"temperature": 0.75, "max_length": 500, "top_p": 1},
                    ) #model-name is from the langchain docs
                    llm_chain = LLMChain(llm=llm, prompt=prompt)

                    stuff_chain = StuffDocumentsChain(
                        llm_chain=llm_chain, document_variable_name="docs")

                    response = stuff_chain.run(
                    user_input="CONVERSATION HISTORY (SYSTEM):" +conversation_history + "USER INPUT:" + user_input,
                    input_documents=docs)

On running this in streamlit, I get an assertionerror:

2024-04-23 10:24:02.386 Uncaught app exception
Traceback (most recent call last):
File "C:\Program Files\Python311\Lib\site-packages\streamlit\runtime\scriptrunner\script_runner.py", line 584, in _run_script
exec(code, module.dict)
File "K:\Python Apps\Pharmacist Advisor\pharm_vdb_frontend_supabase_replIcate.py", line 326, in
main()
File "K:\Python Apps\Pharmacist Advisor\pharm_vdb_frontend_supabase_replIcate.py", line 316, in main
response = stuff_chain.run(
^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain_core_api\deprecation.py", line 145, in warning_emitting_wrapper
return wrapped(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\base.py", line 550, in run
return self(kwargs, callbacks=callbacks, tags=tags, metadata=metadata)[
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain_core_api\deprecation.py", line 145, in warning_emitting_wrapper
return wrapped(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\base.py", line 378, in call
return self.invoke(
^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\base.py", line 163, in invoke
raise e
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\base.py", line 153, in invoke
self._call(inputs, run_manager=run_manager)
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\combine_documents\base.py", line 137, in _call
output, extra_return_dict = self.combine_docs(
^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\combine_documents\stuff.py", line 244, in combine_docs
return self.llm_chain.predict(callbacks=callbacks, **inputs), {}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\llm.py", line 293, in predict
return self(kwargs, callbacks=callbacks)[self.output_key]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain_core_api\deprecation.py", line 145, in warning_emitting_wrapper
return wrapped(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\base.py", line 378, in call
return self.invoke(
^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\base.py", line 163, in invoke
raise e
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\base.py", line 153, in invoke
self._call(inputs, run_manager=run_manager)
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\llm.py", line 103, in _call
response = self.generate([inputs], run_manager=run_manager)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain\chains\llm.py", line 115, in generate
return self.llm.generate_prompt(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain_core\language_models\llms.py", line 569, in generate_prompt
return self.generate(prompt_strings, stop=stop, callbacks=callbacks, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
chain_core\language_models\llms.py", line 1209, in _generate
self._call(prompt, stop=stop, run_manager=run_manager, **kwargs)
File "C:\Program Files\Python311\Lib\site-packages\langchain_community\llms\replicate.py", line 148, in _call
assert completion is not None
^^^^^^^^^^^^^^^^^^^^^^
AssertionError

Is this an error with replicate or langchain? Can you help me with this or do I need to post on the langchain github?

[mattt]

Hi @OlexiyPukhov. This looks like a problem with langchain (I don't see the Replicate package show up in the backtrace). So I'd recommend asking them for help.

One suggestion: You will get better performance if you set the model to "meta/llama-2-13b-chat" — or better yet, one of the new llama 3 models, "meta/meta-llama-3-8b-instruct" or "meta/meta-llama-3-70b-instruct".