Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix file serialization #412

Merged
merged 1 commit into from
Apr 13, 2024
Merged

fix file serialization #412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 4 additions & 21 deletions src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ namespace BotSharp.Core.Conversations.Services;
public class ConversationStorage : IConversationStorage
{
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly BotSharpOptions _options;
private readonly IServiceProvider _services;
private readonly JsonSerializerOptions _jsonOptions;

public ConversationStorage(
BotSharpDatabaseSettings dbSettings,
Expand All @@ -18,7 +18,7 @@ public ConversationStorage(
{
_dbSettings = dbSettings;
_services = services;
_jsonOptions = InitJsonSerilizerOptions(options);
_options = options;
}

public void Append(string conversationId, RoleDialogModel dialog)
Expand Down Expand Up @@ -70,7 +70,7 @@ public void Append(string conversationId, RoleDialogModel dialog)
return;
}

var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _jsonOptions) : null;
var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _options.JsonSerializerOptions) : null;
dialogElements.Add(new DialogElement(meta, content, richContent));
}

Expand All @@ -95,7 +95,7 @@ public List<RoleDialogModel> GetDialogs(string conversationId)
var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
var createdAt = meta.CreateTime;
var richContent = !string.IsNullOrEmpty(dialog.RichContent) ?
JsonSerializer.Deserialize<RichContent<IRichMessage>>(dialog.RichContent, _jsonOptions) : null;
JsonSerializer.Deserialize<RichContent<IRichMessage>>(dialog.RichContent, _options.JsonSerializerOptions) : null;

var record = new RoleDialogModel(role, content)
{
Expand Down Expand Up @@ -140,21 +140,4 @@ private string GetStorageFile(string conversationId)
}
return Path.Combine(dir, "dialogs.txt");
}

private JsonSerializerOptions InitJsonSerilizerOptions(BotSharpOptions botSharOptions)
{
var options = botSharOptions.JsonSerializerOptions;
var jsonOptions = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive,
PropertyNamingPolicy = options.PropertyNamingPolicy ?? JsonNamingPolicy.CamelCase,
AllowTrailingCommas = options.AllowTrailingCommas,
};

foreach (var converter in options.Converters)
{
jsonOptions.Converters.Add(converter);
}
return jsonOptions;
}
}
23 changes: 21 additions & 2 deletions src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ private List<DialogElement> CollectDialogElements(string dialogDir)
CreateTime = DateTime.Parse(blocks[0])
};

var richContent = blocks.Count() > 6 ? blocks[6] : null;
var richContent = blocks.Count() > 6 ? DecodeRichContent(blocks[6]) : null;
dialogs.Add(new DialogElement(meta, trimmed, richContent));
}
}
Expand All @@ -537,7 +537,8 @@ private List<string> ParseDialogElements(List<DialogElement> dialogs)
{
var meta = element.MetaData;
var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.ffffff tt", CultureInfo.InvariantCulture);
var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{meta.SenderId}|{meta.FunctionName}|{element.RichContent}";
var encodedRichContent = EncodeRichContent(element.RichContent);
var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{meta.SenderId}|{meta.FunctionName}|{encodedRichContent}";
dialogTexts.Add(metaStr);
var content = $" - {element.Content}";
dialogTexts.Add(content);
Expand Down Expand Up @@ -686,6 +687,24 @@ private bool SaveTruncatedBreakpoints(string breakpointDir, List<ConversationBre
File.WriteAllText(breakpointDir, breakpointStr);
return true;
}

private string? EncodeRichContent(string? content)
{
if (string.IsNullOrEmpty(content)) return content;

var bytes = Encoding.UTF8.GetBytes(content);
var encoded = Convert.ToBase64String(bytes);
return encoded;
}

private string? DecodeRichContent(string? content)
{
if (string.IsNullOrEmpty(content)) return content;

var decoded = Convert.FromBase64String(content);
var origin = Encoding.UTF8.GetString(decoded);
return origin;
}
#endregion
}
}