Add memory/chat history to a prompt with retrieved nodes processed as context

[Luchao2012]

How does Llamaindex at lower level to include memory/chat history in the RAG process? Does chat history also used for the retrieval of new question?

I have the following customized query engine and prompt, can you instruct me how to add memorty/chat history into the prompt? Also, how to use chat history for the retrival?

my customized prompt:
[
(
"system",
"Return string to the given question using the provided Content and source_name in no particular order as references and use 'source_name' to improve the answer relevance for a given question and also provide citations using ' chunk_id ' to the 'Content' from which you answered the question. "
),
(
"user",
"{context_str}"
),
(
"user",
"You are a professional geologist and petroleum engineer that can answer questions based on the content that I have provided. Please respond to this question : ' {query_str} '. Answer should be properly formatted with all necessary special characters like new line, tab, hyperlinks, image links, and bullets. Answer only if you get the information from the content . If you get partial information then return the partial answer only. Always send relevant and correct chunk_ids with the answer fragments. "
)
]

my customized query engine:
class BaseQueryEngine:
def init(self, index, llm, similarity_top_k=20, use_rerank=True, rerank_top_n=10, text_qa_template_path="prompts/prompt_1.txt", filters=None):
self.index = index
self.similarity_top_k = similarity_top_k
self.text_qa_template = load_prompt(text_qa_template_path)
self.llm = llm
self.filters = filters
self.use_rerank = use_rerank
if self.use_rerank:
#self.reranker = SentenceTransformerRerank(model="cross-encoder/ms-marco-MiniLM-L-2-v2", top_n=rerank_top_n)
self.reranker = CohereRerank(api_key=cohere_api_key, top_n=rerank_top_n)

def query(self, question):
    retriever = self.index.as_retriever(
        similarity_top_k=self.similarity_top_k,
        filters=self.filters)

    retrieved_nodes = retriever.retrieve(question)

    if self.use_rerank:
        logging.info("Using rerank")
        query_bundle = QueryBundle(question)
        reranked_nodes = self.reranker.postprocess_nodes(retrieved_nodes, query_bundle)
    else:
        logging.info("Not using rerank")
        reranked_nodes = retrieved_nodes

    context_str = ""
    source_data = {}
    for i, node in enumerate(reranked_nodes):
        context_str += f"Content: {node.text}. chunk_id: 'chk-{i+1}', source_name: {node.metadata['file_name']}{node.metadata['page_label']}."
        source_data[f"chk-{i+1}"] = {
            "text": node.text,
            "file_name": node.metadata['file_name'],
            "page": node.metadata['page_label']
        }
    context_str = context_str.replace('\n', '')

    fmt_qa_prompt = self.text_qa_template.format(
        context_str=context_str, query_str=question
    )

    response = self.llm.complete(fmt_qa_prompt)

    return response, source_data

class QueryEngineWithTfidfFilter(BaseQueryEngine):
def init(self,
index,
llm,
semantic_weight=0.7,
filter_top_n=5,
similarity_top_k=20,
use_rerank=True,
rerank_top_n=10,
text_qa_template_path="prompts/prompt_1.txt",
file_path_df='metadata_embedding_db.pkl',
embed_model=embed_model
):

    # Initialize the BaseQueryEngine with the necessary parameters
    super().__init__(
        index=index, 
        similarity_top_k=similarity_top_k, 
        text_qa_template_path=text_qa_template_path,  # Correct the parameter name
        llm=llm,
        rerank_top_n=rerank_top_n,
        filters=None,  # Ensure filters is explicitly set if needed
        use_rerank=use_rerank  # Pass the use_rerank parameter
    )
    
    # Additional properties specific to this subclass
    self.metadata_search_engine = MetadataSearchEngine(
        df_path=file_path_df,  # Path to the metadata DataFrame
        embed_model=embed_model  # Embedding model
    )
    self.top_n = filter_top_n
    self.semantic_weight = semantic_weight

def query(self, question):
    # Filter metadata based on the input question
    metadata_search = self.metadata_search_engine.filter(
        question,
        top_n=self.top_n,
        tfidf_weight=1-self.semantic_weight,
    )
    print(f"Metadata filter results: {metadata_search}")
    logging.info(f"Metadata filter results: {metadata_search}")
    # Create MetadataFilters instance using the results from metadata search
    self.filters = MetadataFilters(
        filters=[MetadataFilter(key="ReportFileName", value=item) for item in metadata_search],
        condition=FilterCondition.OR,
    )

    # Call the query method of the base class to perform the query with the updated filters
    return super().query(question)

[dosubot[bot]]

To add memory/chat history into the prompt and use chat history for the retrieval process in your customized query engine and prompt, you can follow these steps:

1. Integrate Chat History into the Prompt:
   Modify your prompt to include the chat history. You can use a placeholder for the chat history and update it dynamically.

2. Use Chat History for Retrieval:
   Ensure that the chat history is considered during the retrieval process by incorporating it into the query.

Here’s how you can modify your code:

Step 1: Modify the Prompt to Include Chat History

Update your prompt to include a placeholder for the chat history:

customized_prompt = [
  (
    "system",
    "Return string to the given question using the provided Content and source_name in no particular order as references and use 'source_name' to improve the answer relevance for a given question and also provide citations using ' chunk_id ' to the 'Content' from which you answered the question. "
  ),
  (
    "user",
    "{chat_history}\n{context_str}"
  ),
  (
    "user",
    "You are a professional geologist and petroleum engineer that can answer questions based on the content that I have provided. Please respond to this question : ' {query_str} '. Answer should be properly formatted with all necessary special characters like new line, tab, hyperlinks, image links, and bullets. Answer only if you get the information from the content . If you get partial information then return the partial answer only. Always send relevant and correct chunk_ids with the answer fragments. "
  )
]

Step 2: Update the Query Engine to Handle Chat History

Modify your BaseQueryEngine class to include chat history in the prompt and use it during retrieval:

class BaseQueryEngine:
    def __init__(self, index, llm, similarity_top_k=20, use_rerank=True, rerank_top_n=10, text_qa_template_path="prompts/prompt_1.txt", filters=None):
        self.index = index
        self.similarity_top_k = similarity_top_k
        self.text_qa_template = load_prompt(text_qa_template_path)
        self.llm = llm
        self.filters = filters
        self.use_rerank = use_rerank
        self.chat_history = []  # Initialize chat history
        if self.use_rerank:
            self.reranker = CohereRerank(api_key=cohere_api_key, top_n=rerank_top_n)

    def query(self, question):
        retriever = self.index.as_retriever(
            similarity_top_k=self.similarity_top_k,
            filters=self.filters)

        # Include chat history in the query
        full_query = "\n".join([msg['content'] for msg in self.chat_history]) + "\n" + question
        retrieved_nodes = retriever.retrieve(full_query)

        if self.use_rerank:
            logging.info("Using rerank")
            query_bundle = QueryBundle(full_query)
            reranked_nodes = self.reranker.postprocess_nodes(retrieved_nodes, query_bundle)
        else:
            logging.info("Not using rerank")
            reranked_nodes = retrieved_nodes

        context_str = ""
        source_data = {}
        for i, node in enumerate(reranked_nodes):
            context_str += f"Content: {node.text}. chunk_id: 'chk-{i+1}', source_name: {node.metadata['file_name']}{node.metadata['page_label']}."
            source_data[f"chk-{i+1}"] = {
                "text": node.text,
                "file_name": node.metadata['file_name'],
                "page": node.metadata['page_label']
            }
        context_str = context_str.replace('\n', '')

        # Format the prompt with chat history
        chat_history_str = "\n".join([f"{msg['role']}: {msg['content']}" for msg in self.chat_history])
        fmt_qa_prompt = self.text_qa_template.format(
            chat_history=chat_history_str, context_str=context_str, query_str=question
        )

        response = self.llm.complete(fmt_qa_prompt)

        # Update chat history
        self.chat_history.append({"role": "user", "content": question})
        self.chat_history.append({"role": "assistant", "content": response})

        return response, source_data

Step 3: Use the Updated Query Engine

When using the QueryEngineWithTfidfFilter class, ensure it inherits the updated behavior from BaseQueryEngine:

class QueryEngineWithTfidfFilter(BaseQueryEngine):
    def __init__(self, 
                 index, 
                 llm,  
                 semantic_weight=0.7, 
                 filter_top_n=5,
                 similarity_top_k=20, 
                 use_rerank=True,
                 rerank_top_n=10,
                 text_qa_template_path="prompts/prompt_1.txt", 
                 file_path_df='metadata_embedding_db.pkl', 
                 embed_model=embed_model
                 ):
        
        super().__init__(
            index=index, 
            similarity_top_k=similarity_top_k, 
            text_qa_template_path=text_qa_template_path,
            llm=llm,
            rerank_top_n=rerank_top_n,
            filters=None,
            use_rerank=use_rerank
        )
        
        self.metadata_search_engine = MetadataSearchEngine(
            df_path=file_path_df,
            embed_model=embed_model
        )
        self.top_n = filter_top_n
        self.semantic_weight = semantic_weight

    def query(self, question):
        metadata_search = self.metadata_search_engine.filter(
            question,
            top_n=self.top_n,
            tfidf_weight=1-self.semantic_weight,
        )
        logging.info(f"Metadata filter results: {metadata_search}")
        self.filters = MetadataFilters(
            filters=[MetadataFilter(key="ReportFileName", value=item) for item in metadata_search],
            condition=FilterCondition.OR,
        )

        return super().query(question)

This setup ensures that the chat history is included in the prompt and used during the retrieval process, enhancing the relevance and context of the responses.

To continue talking to Dosu, mention @dosu.

[metaskills]

I was recently doing something similar running Llama locally with LM Studio but using the Vercel AI SDK and found Zep really helpful in this area. Works with any SDK, Framework, or AI Model. Cheers. https://www.unremarkable.ai/llama-chat-history-with-zeps-ai-memory-knowledge-graph/

[yudin-s]

I would not put the whole chat history directly into the retrieval query. That usually makes retrieval worse after a few turns, because old user phrasing starts competing with the current question.

The pattern that works better is:

1. Use chat history to rewrite the current question into a standalone retrieval query.
2. Retrieve nodes with that rewritten query.
3. Put a short memory summary plus the retrieved context into the final answer prompt.

So the memory has two roles, but they should be separated:

• retrieval memory: only enough to resolve references like “that reservoir” or “the previous report”
• answer memory: a compact summary of user preferences, constraints, and prior decisions

In your custom engine I would add a small step before retriever.retrieve(question):

standalone_question = rewrite_question(chat_history, question)
retrieved_nodes = retriever.retrieve(standalone_question)

Then format the final prompt with:

fmt_qa_prompt = self.text_qa_template.format(
    chat_history=compact_history,
    context_str=context_str,
    query_str=question,
)

Also keep QueryBundle(standalone_question) for reranking, otherwise the reranker is still judging nodes against the ambiguous original question.

One more small thing: avoid context_str.replace('\n', ''). It removes useful structure from technical documents. Prefer joining chunks with clear separators and source metadata, so the model can cite the right chunk more reliably.

[visy-ani]

For folks running into this — maximem-synap-llamaindex (PyPI) exposes two LlamaIndex surfaces that cover this exact pattern: - SynapRetriever (a standard BaseRetriever) — returns typed memory items (facts, preferences, episodes) as NodeWithScore, so retrieved memory drops straight into the same prompt as the rest of your context. - SynapChatMemory (a BaseMemory) — attach to any chat engine to persist conversation across sessions automatically. Open docs PR: #21545 Open source: https://github.com/maximem-ai/maximem_synap_sdk/tree/main/packages/integrations/synap-llamaindex Happy to help if anyone wants a runnable example.

[ferhimedamine]

@yudin-s has the right architecture — separating retrieval memory from answer memory and rewriting the question into a standalone query before retrieval. I want to add one more dimension that makes a significant difference: ranking chat history and retrieved nodes in the same pass instead of concatenating them.

The typical approach is to retrieve nodes from the knowledge base, then prepend some chat history, then send both to the LLM. The problem is that long conversation history pushes out relevant retrieved nodes when you hit context limits, or vice versa. A better approach is to treat conversation history entries as a memory type (episodic) and knowledge base chunks as another type (semantic), then score both in a single ranking pass using a unified function: relevance to the rewritten query multiplied by importance multiplied by recency. Chat messages from two turns ago that are directly relevant to the current question score high. Knowledge base chunks that match the query score high. Old chat messages about unrelated topics score low and get dropped first, regardless of their position in the conversation. This prevents the "context overflow" problem where you have to choose between conversation context and retrieved knowledge.

For the specific implementation in the code above, I would change the retrieval step to also embed and score recent chat history entries alongside the vector-retrieved nodes, then let the Cohere reranker score everything together. The reranker already handles heterogeneous inputs well — it does not care whether a passage came from the vector index or from conversation history, it just scores relevance to the query. This means your top-k results after reranking are the genuinely most relevant pieces of context from both sources combined, rather than a fixed split of N history messages plus M retrieved chunks.

[Ayorinha]

I think retrieval budgets should be evaluated against the context that the model actually receives, not only against the theoretical context window.

In production RAG systems, there can be several layers between retrieval and inference: reranking, filtering, metadata expansion, prompt templates, conversation history, tool results, and output reservations. A nominal context limit therefore does not necessarily represent the effective budget available for retrieved evidence.

I would measure this explicitly with an end-to-end budget such as:

"retrieved tokens → filtered/reranked tokens → prompt tokens → available evidence budget → generated tokens"

More importantly, retrieval quality should be measured at the claim/evidence level. Adding more chunks can increase recall while simultaneously increasing noise, contradiction, and the probability that relevant evidence is effectively ignored.

For each generated claim, I would record which retrieved evidence actually supported it. This makes it possible to distinguish:

• retrieved but unused evidence;
• relevant evidence truncated before inference;
• conflicting evidence;
• unsupported claims;
• and evidence that was available but ignored by the model.

A useful production metric could therefore be an “effective evidence utilization” rate rather than retrieval recall alone.

This also makes RAG evaluation more reproducible: freeze the retrieved set, record the actual prompt/context delivered to the model, and evaluate claim-level grounding against that exact evidence set.

In my experience, the important question is not simply “how much context can the model accept?”, but “how much verified evidence can the model actually use reliably?”

[Ayorinha]

I think retrieval budgets should be evaluated against the context that the model actually receives, not only against the theoretical context window.

In production RAG systems, there can be several layers between retrieval and inference: reranking, filtering, metadata expansion, prompt templates, conversation history, tool results, and output reservations. A nominal context limit therefore does not necessarily represent the effective budget available for retrieved evidence.

I would measure this explicitly with an end-to-end budget such as:

"retrieved tokens → filtered/reranked tokens → prompt tokens → available evidence budget → generated tokens"

More importantly, retrieval quality should be measured at the claim/evidence level. Adding more chunks can increase recall while simultaneously increasing noise, contradiction, and the probability that relevant evidence is effectively ignored.

For each generated claim, I would record which retrieved evidence actually supported it. This makes it possible to distinguish:

• retrieved but unused evidence;
• relevant evidence truncated before inference;
• conflicting evidence;
• unsupported claims;
• and evidence that was available but ignored by the model.

A useful production metric could therefore be an “effective evidence utilization” rate rather than retrieval recall alone.

This also makes RAG evaluation more reproducible: freeze the retrieved set, record the actual prompt/context delivered to the model, and evaluate claim-level grounding against that exact evidence set.

In my experience, the important question is not simply “how much context can the model accept?”, but “how much verified evidence can the model actually use reliably?”

[Icaro0310]

Don't inject history into the prompt manually. Wrap your query engine in a chat engine. CondensePlusContextChatEngine keeps a ChatMemoryBuffer, and on each turn it rewrites the follow-up into a standalone question using the history, then runs that against your retriever/query engine. That's exactly how chat history feeds retrieval.

from llama_index.core.memory import ChatMemoryBuffer
from llama_index.core.chat_engine import CondensePlusContextChatEngine

memory = ChatMemoryBuffer.from_defaults(token_limit=3000)
chat_engine = CondensePlusContextChatEngine.from_defaults(
 query_engine=query_engine
 memory=memory
 system_prompt="Return string to t..."
)

response = chat_engine.chat("your question")

So history is used twice: once to condense the question before retrieval, and again as conversation context for the final LLM call. If you truly want a chat_history variable inside your own message template, ContextChatEngine exposes it, but the wrapper above covers your case with less plumbing.

—
Related work of mine: github.com/Icaro0310 · Website: icaro0310.github.io

[VimalN2005]

Solution: Integrating ChatMemoryBuffer & Query Condensation in LlamaIndex

In LlamaIndex, conversational RAG with memory requires two components:

1. ChatMemoryBuffer: Stores message history within a token budget.
2. Context / Condense Engine: Uses history to rewrite follow-up questions before retrieval, then passes both history and context nodes to the prompt.

---

Cleanest Native Approach: as_chat_engine

Instead of manually parsing message strings in a custom query engine, use LlamaIndex's built-in condense_plus_context engine:

from llama_index.core.memory import ChatMemoryBuffer

# 1. Initialize Memory Buffer
memory = ChatMemoryBuffer.from_defaults(token_limit=3000)

# 2. Create Chat Engine from your Index
chat_engine = index.as_chat_engine(
    chat_mode="condense_plus_context",
    memory=memory,
    similarity_top_k=5,
    system_prompt=(
        "You are a professional geologist and petroleum engineer. "
        "Answer strictly based on the provided context."
    ),
    verbose=True
)

# Multi-turn conversation:
response1 = chat_engine.chat("What is the permeability of reservoir A?")
response2 = chat_engine.chat("How does that compare to reservoir B?")  # Automatically resolves context!

---

How to do it inside your Custom BaseQueryEngine

If you must maintain your custom class, inject ChatMemoryBuffer and format history messages into the prompt:

from llama_index.core.memory import ChatMemoryBuffer
from llama_index.core.llms import ChatMessage, MessageRole

class CustomConversationalEngine:
    def __init__(self, index, llm):
        self.index = index
        self.llm = llm
        self.memory = ChatMemoryBuffer.from_defaults(token_limit=3000)
    
    def chat(self, user_question: str):
        # 1. Retrieve nodes
        retriever = self.index.as_retriever(similarity_top_k=5)
        nodes = retriever.retrieve(user_question)
        context_str = "\n".join([f"Content: {n.text}" for n in nodes])
        
        # 2. Format Messages
        messages = [
            ChatMessage(role=MessageRole.SYSTEM, content="You are a domain expert. Use context to answer."),
            *self.memory.get(),  # Injects previous conversation turns
            ChatMessage(role=MessageRole.USER, content=f"Context:\n{context_str}\n\nQuestion: {user_question}")
        ]
        
        # 3. Call Chat
        response = self.llm.chat(messages)
        
        # 4. Save turn to memory
        self.memory.put(ChatMessage(role=MessageRole.USER, content=user_question))
        self.memory.put(ChatMessage(role=MessageRole.ASSISTANT, content=str(response)))
        
        return response

If this helps structure your conversational RAG pipeline, please mark as accepted answer!