Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit fb551b7

Browse files
OkGoDoItOkGoDoIt
committed
Add auto-truncation for chat conversation history when it exceeds the token limit
1 parent 8bc534b commit fb551b7

File tree

4 files changed

+753
-336
lines changed

4 files changed

+753
-336
lines changed

‎OpenAI_API/Chat/Conversation.cs‎

Lines changed: 169 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.Data;
44
using System.Linq;
5+
using System.Net.Http;
56
using System.Text;
67
using System.Threading;
78
using System.Threading.Tasks;
@@ -111,6 +112,16 @@ public void AppendMessage(ChatMessage message)
111112
/// <param name="content">Text content written by a developer to help give examples of desired behavior</param>
112113
public void AppendExampleChatbotOutput(string content) => this.AppendMessage(new ChatMessage(ChatMessageRole.Assistant, content));
113114

115+
/// <summary>
116+
/// An event called when the chat message history is too long, which should reduce message history length through whatever means is appropriate for your use case. You may want to remove the first entry in the <see cref="List{ChatMessage}"/> in the <see cref="EventArgs"/>
117+
/// </summary>
118+
public event EventHandler<List<ChatMessage>> OnTruncationNeeded;
119+
120+
/// <summary>
121+
/// Sometimes the total length of your conversation can get too long to fit in the ChatGPT context window. In this case, the <see cref="OnTruncationNeeded"/> event will be called, if supplied. If not supplied and this is <see langword="true"/>, then the first one or more user or assistant messages will be automatically deleted from the beginning of the conversation message history until the API call succeeds. This may take some time as it may need to loop several times to clear enough messages. If this is set to false and no <see cref="OnTruncationNeeded"/> is supplied, then an <see cref="ArgumentOutOfRangeException"/> will be raised when the API returns a context_length_exceeded error.
122+
/// </summary>
123+
public bool AutoTruncateOnContextLengthExceeded { get; set; } = true;
124+
114125
#region Non-streaming
115126

116127
/// <summary>
@@ -119,19 +130,70 @@ public void AppendMessage(ChatMessage message)
119130
/// <returns>The string of the response from the chatbot API</returns>
120131
public async Task<string> GetResponseFromChatbotAsync()
121132
{
122-
ChatRequest req = new ChatRequest(RequestParameters);
123-
req.Messages = _Messages.ToList();
133+
try
134+
{
135+
ChatRequest req = new ChatRequest(RequestParameters);
136+
req.Messages = _Messages.ToList();
124137

125-
var res = await _endpoint.CreateChatCompletionAsync(req);
126-
MostRecentApiResult = res;
138+
var res = await _endpoint.CreateChatCompletionAsync(req);
139+
MostRecentApiResult = res;
127140

128-
if (res.Choices.Count > 0)
141+
if (res.Choices.Count > 0)
142+
{
143+
var newMsg = res.Choices[0].Message;
144+
AppendMessage(newMsg);
145+
return newMsg.Content;
146+
}
147+
}
148+
catch (HttpRequestException ex)
129149
{
130-
var newMsg = res.Choices[0].Message;
131-
AppendMessage(newMsg);
132-
return newMsg.Content;
150+
if (ex.Data.Contains("code") && (!string.IsNullOrEmpty(ex.Data["code"] as string)) && ex.Data["code"].Equals("context_length_exceeded"))
151+
{
152+
string message = "The context length of this conversation is too long for the OpenAI API to handle. Consider shortening the message history by handling the OnTruncationNeeded event and removing some of the messages in the argument.";
153+
if (ex.Data.Contains("message"))
154+
{
155+
message += " " + ex.Data["message"].ToString();
156+
}
157+
158+
if (OnTruncationNeeded != null)
159+
{
160+
var prevLength = this.Messages.Sum(m => m.Content.Length);
161+
OnTruncationNeeded(this, this._Messages);
162+
if (prevLength > this.Messages.Sum(m => m.Content.Length))
163+
{
164+
// the messages have been truncated, so try again
165+
return await GetResponseFromChatbotAsync();
166+
}
167+
else
168+
{
169+
// no truncation happened, so throw error instead
170+
throw new ArgumentOutOfRangeException("OnTruncationNeeded was called but it did not reduce the message history length. " + message, ex);
171+
}
172+
}
173+
else if (AutoTruncateOnContextLengthExceeded)
174+
{
175+
for (int i = 0; i < _Messages.Count; i++)
176+
{
177+
if (_Messages[i].Role != ChatMessageRole.System)
178+
{
179+
_Messages.RemoveAt(i);
180+
// the messages have been truncated, so try again
181+
return await GetResponseFromChatbotAsync();
182+
}
183+
}
184+
}
185+
else
186+
{
187+
throw new ArgumentOutOfRangeException(message, ex);
188+
}
189+
}
190+
else
191+
{
192+
throw ex;
193+
}
133194
}
134195
return null;
196+
135197
}
136198

137199
/// <summary>
@@ -180,13 +242,109 @@ public async Task StreamResponseFromChatbotAsync(Action<int, string> resultHandl
180242
/// <returns>An async enumerable with each of the results as they come in. See <see href="https://docs.microsoft.com/en-us/dotnet/csharp/whats-new/csharp-8#asynchronous-streams"/> for more details on how to consume an async enumerable.</returns>
181243
public async IAsyncEnumerable<string> StreamResponseEnumerableFromChatbotAsync()
182244
{
183-
ChatRequest req = new ChatRequest(RequestParameters);
184-
req.Messages = _Messages.ToList();
245+
ChatRequest req = null;
185246

186247
StringBuilder responseStringBuilder = new StringBuilder();
187248
ChatMessageRole responseRole = null;
188249

189-
await foreach (var res in _endpoint.StreamChatEnumerableAsync(req))
250+
IAsyncEnumerable<ChatResult> resStream = null;
251+
252+
bool retrying = true;
253+
ChatResult firstStreamedResult = null;
254+
255+
while (retrying)
256+
{
257+
retrying = false;
258+
req = new ChatRequest(RequestParameters);
259+
req.Messages = _Messages.ToList();
260+
try
261+
{
262+
resStream = _endpoint.StreamChatEnumerableAsync(req);
263+
await foreach (var res in resStream)
264+
{
265+
if (res != null)
266+
{
267+
firstStreamedResult = res;
268+
break;
269+
}
270+
}
271+
}
272+
catch (HttpRequestException ex)
273+
{
274+
if (ex.Data.Contains("code") && (!string.IsNullOrEmpty(ex.Data["code"] as string)) && ex.Data["code"].Equals("context_length_exceeded"))
275+
{
276+
string message = "The context length of this conversation is too long for the OpenAI API to handle. Consider shortening the message history by handling the OnTruncationNeeded event and removing some of the messages in the argument.";
277+
if (ex.Data.Contains("message"))
278+
{
279+
message += " " + ex.Data["message"].ToString();
280+
}
281+
282+
if (OnTruncationNeeded != null)
283+
{
284+
var prevLength = this.Messages.Sum(m => m.Content.Length);
285+
OnTruncationNeeded(this, this._Messages);
286+
if (prevLength > this.Messages.Sum(m => m.Content.Length))
287+
{
288+
// the messages have been truncated, so try again
289+
retrying = true;
290+
}
291+
else
292+
{
293+
// no truncation happened, so throw error instead
294+
retrying = false;
295+
throw new ArgumentOutOfRangeException("OnTruncationNeeded was called but it did not reduce the message history length. " + message, ex);
296+
}
297+
}
298+
else if (AutoTruncateOnContextLengthExceeded)
299+
{
300+
for (int i = 0; i < _Messages.Count; i++)
301+
{
302+
if (_Messages[i].Role != ChatMessageRole.System)
303+
{
304+
_Messages.RemoveAt(i);
305+
// the messages have been truncated, so try again
306+
retrying = true;
307+
break;
308+
}
309+
}
310+
}
311+
else
312+
{
313+
retrying = false;
314+
throw new ArgumentOutOfRangeException(message, ex);
315+
}
316+
}
317+
else
318+
{
319+
throw ex;
320+
}
321+
}
322+
}
323+
324+
if (resStream == null)
325+
{
326+
throw new Exception("The chat result stream is null, but it shouldn't be");
327+
}
328+
329+
if (firstStreamedResult != null)
330+
{
331+
if (firstStreamedResult.Choices.FirstOrDefault()?.Delta is ChatMessage delta)
332+
{
333+
if (delta.Role != null)
334+
responseRole = delta.Role;
335+
336+
string deltaContent = delta.Content;
337+
338+
if (!string.IsNullOrEmpty(deltaContent))
339+
{
340+
responseStringBuilder.Append(deltaContent);
341+
yield return deltaContent;
342+
}
343+
}
344+
MostRecentApiResult = firstStreamedResult;
345+
}
346+
347+
await foreach (var res in resStream)
190348
{
191349
if (res.Choices.FirstOrDefault()?.Delta is ChatMessage delta)
192350
{

0 commit comments

Comments
 (0)