Skip to content

Commit 0a85195

Browse files
author
Jicheng Lu
committed
refine global stats schema
1 parent 6141643 commit 0a85195

File tree

10 files changed

+131
-62
lines changed

10 files changed

+131
-62
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,11 @@ public class BotSharpStats
77
[JsonPropertyName("agent_id")]
88
public string AgentId { get; set; } = null!;
99

10-
[JsonPropertyName("agent_call_count")]
11-
public int AgentCallCount { get; set; }
12-
13-
[JsonPropertyName("prompt_tokens")]
14-
public int PromptTokens { get; set; }
15-
16-
[JsonPropertyName("completion_tokens")]
17-
public int CompletionTokens { get; set; }
18-
19-
[JsonPropertyName("prompt_total_cost")]
20-
public float PromptTotalCost { get; set; }
10+
[JsonPropertyName("count")]
11+
public StatsCount Count { get; set; } = new();
2112

22-
[JsonPropertyName("completion_total_cost")]
23-
public float CompletionTotalCost { get; set; }
13+
[JsonPropertyName("llm_cost")]
14+
public StatsLlmCost LlmCost { get; set; } = new();
2415

2516
[JsonPropertyName("record_time")]
2617
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
@@ -83,4 +74,25 @@ public static (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsI
8374
endTime = DateTime.SpecifyKind(endTime, DateTimeKind.Utc);
8475
return (startTime, endTime);
8576
}
77+
}
78+
79+
public class StatsCount
80+
{
81+
[JsonPropertyName("agent_call_count")]
82+
public long AgentCallCount { get; set; }
83+
}
84+
85+
public class StatsLlmCost
86+
{
87+
[JsonPropertyName("prompt_tokens")]
88+
public long PromptTokens { get; set; }
89+
90+
[JsonPropertyName("completion_tokens")]
91+
public long CompletionTokens { get; set; }
92+
93+
[JsonPropertyName("prompt_total_cost")]
94+
public float PromptTotalCost { get; set; }
95+
96+
[JsonPropertyName("completion_total_cost")]
97+
public float CompletionTotalCost { get; set; }
8698
}

‎src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,8 @@ namespace BotSharp.Abstraction.Statistics.Models;
55
public class BotSharpStatsDelta
66
{
77
public string AgentId { get; set; } = null!;
8-
public int AgentCallCountDelta { get; set; }
9-
public int PromptTokensDelta { get; set; }
10-
public int CompletionTokensDelta { get; set; }
11-
public float PromptTotalCostDelta { get; set; }
12-
public float CompletionTotalCostDelta { get; set; }
8+
public StatsCountDelta CountDelta { get; set; } = new();
9+
public StatsLlmCostDelta LlmCostDelta { get; set; } = new();
1310
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
1411
public StatsInterval IntervalType { get; set; } = StatsInterval.Day;
1512

@@ -28,3 +25,16 @@ public string Interval
2825
}
2926
}
3027
}
28+
29+
public class StatsCountDelta
30+
{
31+
public int AgentCallCountDelta { get; set; }
32+
}
33+
34+
public class StatsLlmCostDelta
35+
{
36+
public int PromptTokensDelta { get; set; }
37+
public int CompletionTokensDelta { get; set; }
38+
public float PromptTotalCostDelta { get; set; }
39+
public float CompletionTotalCostDelta { get; set; }
40+
}

‎src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,13 @@ public void AddToken(TokenStatsModel stats, RoleDialogModel message)
7878
AgentId = agentId,
7979
RecordTime = DateTime.UtcNow,
8080
IntervalType = StatsInterval.Day,
81-
PromptTokensDelta = stats.TotalInputTokens,
82-
CompletionTokensDelta = stats.TotalOutputTokens,
83-
PromptTotalCostDelta = deltaPromptCost,
84-
CompletionTotalCostDelta = deltaCompletionCost
81+
LlmCostDelta = new()
82+
{
83+
PromptTokensDelta = stats.TotalInputTokens,
84+
CompletionTokensDelta = stats.TotalOutputTokens,
85+
PromptTotalCostDelta = deltaPromptCost,
86+
CompletionTotalCostDelta = deltaCompletionCost
87+
}
8588
};
8689
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta);
8790
}

‎src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,17 @@ public bool SaveGlobalStats(BotSharpStatsDelta delta)
5353
var newItem = new BotSharpStats
5454
{
5555
AgentId = delta.AgentId,
56-
AgentCallCount = delta.AgentCallCountDelta,
57-
PromptTokens = delta.PromptTokensDelta,
58-
CompletionTokens = delta.CompletionTokensDelta,
59-
PromptTotalCost = delta.PromptTotalCostDelta,
60-
CompletionTotalCost = delta.CompletionTotalCostDelta,
56+
Count = new()
57+
{
58+
AgentCallCount = delta.CountDelta.AgentCallCountDelta
59+
},
60+
LlmCost = new()
61+
{
62+
PromptTokens = delta.LlmCostDelta.PromptTokensDelta,
63+
CompletionTokens = delta.LlmCostDelta.CompletionTokensDelta,
64+
PromptTotalCost = delta.LlmCostDelta.PromptTotalCostDelta,
65+
CompletionTotalCost = delta.LlmCostDelta.CompletionTotalCostDelta,
66+
},
6167
RecordTime = delta.RecordTime,
6268
StartTime = startTime,
6369
EndTime = endTime,
@@ -82,11 +88,11 @@ public bool SaveGlobalStats(BotSharpStatsDelta delta)
8288
{
8389
found.AgentId = delta.AgentId;
8490
found.RecordTime = delta.RecordTime;
85-
found.AgentCallCount += delta.AgentCallCountDelta;
86-
found.PromptTokens += delta.PromptTokensDelta;
87-
found.CompletionTokens += delta.CompletionTokensDelta;
88-
found.PromptTotalCost += delta.PromptTotalCostDelta;
89-
found.CompletionTotalCost += delta.CompletionTotalCostDelta;
91+
found.Count.AgentCallCount += delta.CountDelta.AgentCallCountDelta;
92+
found.LlmCost.PromptTokens += delta.LlmCostDelta.PromptTokensDelta;
93+
found.LlmCost.CompletionTokens += delta.LlmCostDelta.CompletionTokensDelta;
94+
found.LlmCost.PromptTotalCost += delta.LlmCostDelta.PromptTotalCostDelta;
95+
found.LlmCost.CompletionTotalCost += delta.LlmCostDelta.CompletionTotalCostDelta;
9096
found.StartTime = startTime;
9197
found.EndTime = endTime;
9298
found.Interval = delta.Interval;

‎src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@ private void UpdateAgentCall(RoleDialogModel message)
3333
AgentId = agentId,
3434
RecordTime = DateTime.UtcNow,
3535
IntervalType = StatsInterval.Day,
36-
AgentCallCountDelta = 1
36+
CountDelta = new()
37+
{
38+
AgentCallCountDelta = 1
39+
}
3740
};
3841
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta);
3942
}

‎src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,8 @@ namespace BotSharp.Plugin.MongoStorage.Collections;
33
public class GlobalStatisticsDocument : MongoBase
44
{
55
public string AgentId { get; set; } = null!;
6-
public int AgentCallCount { get; set; }
7-
public int PromptTokens { get; set; }
8-
public int CompletionTokens { get; set; }
9-
public float PromptTotalCost { get; set; }
10-
public float CompletionTotalCost { get; set; }
6+
public StatsCountMongoElement Count { get; set; } = new();
7+
public StatsLlmCostMongoElement LlmCost { get; set; } = new();
118

129
public DateTime RecordTime { get; set; }
1310
public DateTime StartTime { get; set; }

‎src/Plugins/BotSharp.Plugin.MongoStorage/Models/GlobalStatsMongoElement.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
namespace BotSharp.Plugin.MongoStorage.Models;
2+
3+
public class StatsCountMongoElement
4+
{
5+
public long AgentCallCount { get; set; }
6+
}
7+
8+
public class StatsLlmCostMongoElement
9+
{
10+
public long PromptTokens { get; set; }
11+
public long CompletionTokens { get; set; }
12+
public float PromptTotalCost { get; set; }
13+
public float CompletionTotalCost { get; set; }
14+
}

‎src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,8 +203,8 @@ public IMongoCollection<RoleAgentDocument> RoleAgents
203203
public IMongoCollection<CrontabItemDocument> CrontabItems
204204
=> GetCollectionOrCreate<CrontabItemDocument>("CronTabItems");
205205

206-
public IMongoCollection<GlobalStatisticsDocument> GlobalStatistics
207-
=> GetCollectionOrCreate<GlobalStatisticsDocument>("GlobalStatistics");
206+
public IMongoCollection<GlobalStatisticsDocument> GlobalStats
207+
=> GetCollectionOrCreate<GlobalStatisticsDocument>("GlobalStats");
208208

209209
public IMongoCollection<InstructionLogDocument> InstructionLogs
210210
=> CreateInstructionLogIndex();

‎src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,22 @@ public partial class MongoRepository
2323
};
2424

2525
var filterDef = builder.And(filters);
26-
var found = _dc.GlobalStatistics.Find(filterDef).FirstOrDefault();
26+
var found = _dc.GlobalStats.Find(filterDef).FirstOrDefault();
2727

2828
return found != null ? new BotSharpStats
2929
{
3030
AgentId = agentId,
31-
AgentCallCount = found.AgentCallCount,
32-
PromptTokens = found.PromptTokens,
33-
CompletionTokens = found.CompletionTokens,
34-
PromptTotalCost = found.PromptTotalCost,
35-
CompletionTotalCost = found.CompletionTotalCost,
31+
Count = new()
32+
{
33+
AgentCallCount = found.Count.AgentCallCount
34+
},
35+
LlmCost = new()
36+
{
37+
PromptTokens = found.LlmCost.PromptTokens,
38+
CompletionTokens = found.LlmCost.CompletionTokens,
39+
PromptTotalCost = found.LlmCost.PromptTotalCost,
40+
CompletionTotalCost = found.LlmCost.CompletionTotalCost
41+
},
3642
RecordTime = found.RecordTime,
3743
StartTime = startTime,
3844
EndTime = endTime,
@@ -61,17 +67,17 @@ public bool SaveGlobalStats(BotSharpStatsDelta delta)
6167
var filterDef = builder.And(filters);
6268
var updateDef = Builders<GlobalStatisticsDocument>.Update
6369
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
64-
.Inc(x => x.AgentCallCount, delta.AgentCallCountDelta)
65-
.Inc(x => x.PromptTokens, delta.PromptTokensDelta)
66-
.Inc(x => x.CompletionTokens, delta.CompletionTokensDelta)
67-
.Inc(x => x.PromptTotalCost, delta.PromptTotalCostDelta)
68-
.Inc(x => x.CompletionTotalCost, delta.CompletionTotalCostDelta)
70+
.Inc(x => x.Count.AgentCallCount, delta.CountDelta.AgentCallCountDelta)
71+
.Inc(x => x.LlmCost.PromptTokens, delta.LlmCostDelta.PromptTokensDelta)
72+
.Inc(x => x.LlmCost.CompletionTokens, delta.LlmCostDelta.CompletionTokensDelta)
73+
.Inc(x => x.LlmCost.PromptTotalCost, delta.LlmCostDelta.PromptTotalCostDelta)
74+
.Inc(x => x.LlmCost.CompletionTotalCost, delta.LlmCostDelta.CompletionTotalCostDelta)
6975
.Set(x => x.StartTime, startTime)
7076
.Set(x => x.EndTime, endTime)
7177
.Set(x => x.Interval, delta.Interval)
7278
.Set(x => x.RecordTime, delta.RecordTime);
7379

74-
_dc.GlobalStatistics.UpdateOne(filterDef, updateDef, _options);
80+
_dc.GlobalStats.UpdateOne(filterDef, updateDef, _options);
7581
return true;
7682
}
7783
}

‎src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using BotSharp.Abstraction.Agents.Models;
12
using OpenAI.Chat;
23

34
namespace BotSharp.Plugin.OpenAI.Providers.Chat;
@@ -11,6 +12,11 @@ public class ChatCompletionProvider : IChatCompletion
1112
protected string _model;
1213
private List<string> renderedInstructions = [];
1314

15+
private readonly Dictionary<string, float> _defaultTemperature = new()
16+
{
17+
{ "o4-mini", 1.0f }
18+
};
19+
1420
public virtual string Provider => "openai";
1521
public string Model => _model;
1622

@@ -220,16 +226,7 @@ await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessa
220226
renderedInstructions = [];
221227

222228
var messages = new List<ChatMessage>();
223-
224-
var temperature = float.Parse(state.GetState("temperature", "0.0"));
225-
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
226-
? tokens
227-
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
228-
var options = new ChatCompletionOptions()
229-
{
230-
Temperature = temperature,
231-
MaxOutputTokenCount = maxTokens
232-
};
229+
var options = InitChatCompletionOption(agent);
233230

234231
var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
235232
foreach (var function in functions)
@@ -391,6 +388,27 @@ private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOption
391388
return prompt;
392389
}
393390

391+
private ChatCompletionOptions InitChatCompletionOption(Agent agent)
392+
{
393+
var state = _services.GetRequiredService<IConversationStateService>();
394+
395+
var temperature = float.Parse(state.GetState("temperature", "0.0"));
396+
if (_defaultTemperature.ContainsKey(_model))
397+
{
398+
temperature = _defaultTemperature[_model];
399+
}
400+
401+
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
402+
? tokens
403+
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
404+
405+
return new ChatCompletionOptions()
406+
{
407+
Temperature = temperature,
408+
MaxOutputTokenCount = maxTokens
409+
};
410+
}
411+
394412
public void SetModelName(string model)
395413
{
396414
_model = model;

0 commit comments

Comments
 (0)