← Back to Files
Program.cs
using System.Globalization;
using System.Diagnostics;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
var options = Options.Parse(args);
var baseDir = Path.GetFullPath(options.BaseDir);
var sourceDir = Path.GetFullPath(Path.Combine(baseDir, options.SourceDir));
var outputRoot = Path.GetFullPath(Path.Combine(baseDir, options.OutputDir));
var latestDir = Path.Combine(outputRoot, "latest");
var snapshotsDir = Path.Combine(outputRoot, "snapshots");
var checkpointDir = Path.Combine(outputRoot, "_checkpoints");
var historyCheckpointDir = Path.Combine(outputRoot, "_history_checkpoints");
var historyFragmentDir = Path.Combine(outputRoot, "_history_fragments");
var historyFragmentCheckpointDir = Path.Combine(outputRoot, "_history_fragment_checkpoints");
List<SourceConversation> sources = !string.IsNullOrWhiteSpace(options.SourceFile)
? [SourceConversation.FromAdHocPath(Path.GetFullPath(Path.Combine(baseDir, options.SourceFile)))]
: BuildOrderedSources(sourceDir);
if ((options.HistoryPass || options.HistoryFragmentPass)
&& string.IsNullOrWhiteSpace(options.SourceFile)
&& !string.IsNullOrWhiteSpace(options.IncludeCurrentSourceFile))
{
sources.Add(SourceConversation.FromAdHocPath(Path.GetFullPath(Path.Combine(baseDir, options.IncludeCurrentSourceFile))));
}
if (string.IsNullOrWhiteSpace(options.SourceFile) && !Directory.Exists(sourceDir))
throw new DirectoryNotFoundException(sourceDir);
foreach (var source in sources)
{
if (!File.Exists(source.Path))
throw new FileNotFoundException($"Source file not found: {source.Path}", source.Path);
}
Console.WriteLine($"Source: {(string.IsNullOrWhiteSpace(options.SourceFile) ? sourceDir : sources[0].Path)}");
Console.WriteLine($"Output root: {outputRoot}");
Console.WriteLine($"Latest memory: {latestDir}");
Console.WriteLine($"Snapshots: {snapshotsDir}");
Console.WriteLine($"Mode: {(options.Apply ? "apply" : "dry run")}");
Console.WriteLine($"History pass: {(options.HistoryPass ? "on" : "off")}");
Console.WriteLine($"History fragment pass: {(options.HistoryFragmentPass ? "on" : "off")}");
Console.WriteLine($"History merge pass: {(options.HistoryMergePass ? "on" : "off")}");
Console.WriteLine($"Chunk target: {options.ChunkTokens:N0} tokens (~{options.MaxChunkChars:N0} chars)");
Console.WriteLine($"Max output: {options.MaxOutputTokens:N0} tokens");
Console.WriteLine($"Request idle timeout: {(options.RequestIdleTimeoutMinutes > 0 ? $"{options.RequestIdleTimeoutMinutes:N0} minute(s)" : "off")}");
Console.WriteLine($"Retention audit: {(options.RetentionAudit ? "on" : "off")}");
Console.WriteLine($"Change audit: {(options.ChangeAudit ? "on" : "off")}");
Console.WriteLine($"Sources: {sources.Count}");
Console.WriteLine();
var selected = sources
.Where(s => SourceSelected(s, options))
.Where(s => !options.SkipSnapshotted || !SnapshotExists(snapshotsDir, s))
.ToList();
if (!string.IsNullOrWhiteSpace(options.SourceKey) && selected.Count == 0)
throw new InvalidOperationException($"No source found for --source-key {options.SourceKey}");
if (options.Apply)
{
Directory.CreateDirectory(outputRoot);
Directory.CreateDirectory(latestDir);
Directory.CreateDirectory(snapshotsDir);
Directory.CreateDirectory(checkpointDir);
if (options.HistoryPass)
Directory.CreateDirectory(historyCheckpointDir);
if (options.HistoryFragmentPass)
{
Directory.CreateDirectory(historyFragmentDir);
Directory.CreateDirectory(historyFragmentCheckpointDir);
}
}
var client = options.Apply ? await ModelClient.CreateAsync(baseDir, options) : null;
var processedChunks = 0;
if (options.HistoryFragmentPass)
{
await RunHistoryFragmentPassAsync(
selected,
options,
client,
historyCheckpointDir,
historyFragmentCheckpointDir,
historyFragmentDir,
baseDir);
return;
}
if (options.HistoryMergePass)
{
await RunHistoryMergePassAsync(options, client, latestDir, historyFragmentDir);
return;
}
if (options.HistoryPass)
{
await RunHistoryPassAsync(selected, options, client, latestDir, historyCheckpointDir, baseDir);
return;
}
foreach (var source in selected)
{
var text = await ReadSourceTranscriptAsync(source);
var blocks = ExtractMessageBlocks(text);
var chunks = BuildChunks(blocks, options.MaxChunkChars);
var stopAfterCurrentSource = false;
var sourceStartMemory = options.Apply ? await ReadMemoryDirectoryAsync(latestDir) : "";
Console.WriteLine($"{source.DisplayName}: {blocks.Count:N0} message blocks -> {chunks.Count:N0} chunks");
for (var i = 0; i < chunks.Count; i++)
{
var checkpointPath = Path.Combine(checkpointDir, $"{source.Key}_chunk_{i + 1:D3}.json");
if (options.Resume && File.Exists(checkpointPath))
{
Console.WriteLine($" skip chunk {i + 1}/{chunks.Count} (checkpoint exists)");
continue;
}
var chunkText = string.Join("\n\n", chunks[i]);
Console.WriteLine($" chunk {i + 1}/{chunks.Count}: ~{EstimateTokens(chunkText):N0} tokens");
if (options.Apply)
{
var beforeMemory = await ReadMemoryDirectoryAsync(latestDir);
var prompt = BuildPrompt(source, i + 1, chunks.Count, chunkText, beforeMemory);
var raw = await client!.ChatAsync(prompt);
var rawResponsePath = Path.Combine(checkpointDir, $"{source.Key}_chunk_{i + 1:D3}_primary.raw.txt");
await File.WriteAllTextAsync(rawResponsePath, raw);
var parseResult = await ParseOrRepairChangeSetAsync(client, raw, rawResponsePath);
var changeSet = parseResult.ChangeSet;
var operationResults = new List<OperationResult>();
foreach (var op in changeSet.Operations)
operationResults.Add(await ApplyOperationAsync(latestDir, op));
RetentionAuditCheckpoint? retentionAudit = null;
if (options.RetentionAudit && !IsEmptyMemoryDirectoryText(beforeMemory))
{
retentionAudit = await RunRetentionAuditAsync(
client,
latestDir,
source,
i + 1,
chunks.Count,
chunkText,
beforeMemory,
checkpointDir);
if (retentionAudit.Operations.Any(r => r.Success && r.Action is "save" or "write" or "upsert"))
Console.WriteLine($" retention audit repaired {retentionAudit.Operations.Count(r => r.Success && r.Action is "save" or "write" or "upsert"):N0} file(s)");
}
var checkpoint = new ChunkCheckpoint(
Source: source.DisplayName,
SourcePath: Path.GetRelativePath(baseDir, source.Path).Replace('\\', '/'),
ChunkIndex: i + 1,
ChunkCount: chunks.Count,
EstimatedTokens: EstimateTokens(chunkText),
CreatedAtUtc: DateTime.UtcNow,
RawResponse: raw,
JsonRepair: parseResult.Repair,
Operations: operationResults,
RetentionAudit: retentionAudit);
await File.WriteAllTextAsync(
checkpointPath,
JsonSerializer.Serialize(checkpoint, Shared.JsonOptions) + Environment.NewLine);
CleanupParseTempFiles(checkpointDir, rawResponsePath, parseResult.Repair);
if (retentionAudit is not null)
CleanupParseTempFiles(checkpointDir, retentionAudit.RawResponsePath, retentionAudit.JsonRepair);
}
processedChunks++;
if (options.MaxChunks > 0 && processedChunks >= options.MaxChunks)
{
if (i + 1 < chunks.Count)
{
Console.WriteLine();
Console.WriteLine($"Stopped after --max-chunks {options.MaxChunks}.");
return;
}
stopAfterCurrentSource = true;
break;
}
}
if (options.Apply)
{
SourceChangeAuditReport? changeAuditReport = null;
if (options.ChangeAudit)
{
var sourceEndMemory = await ReadMemoryDirectoryAsync(latestDir);
changeAuditReport = await RunSourceChangeAuditAsync(
client!,
checkpointDir,
source,
sourceStartMemory,
sourceEndMemory);
Console.WriteLine($" change audit: {changeAuditReport.Added.Count:N0} added, {changeAuditReport.Changed.Count:N0} changed, {changeAuditReport.Moved.Count:N0} moved, {changeAuditReport.Removed.Count:N0} removed");
var lossRiskCount = CountLossRisks(changeAuditReport);
if (lossRiskCount > 0)
Console.WriteLine($" change audit WARNING: {lossRiskCount:N0} possible unjustified loss item(s)");
}
var snapshotDir = await SnapshotLatestAsync(latestDir, snapshotsDir, source);
if (changeAuditReport is not null)
await WriteSourceChangeAuditReportAsync(snapshotDir, changeAuditReport);
Console.WriteLine($" snapshot: {Path.GetRelativePath(outputRoot, snapshotDir).Replace('\\', '/')}");
if (options.CommitAfterSource)
await CommitRebuildStateAsync(baseDir, outputRoot, source, changeAuditReport);
}
if (stopAfterCurrentSource)
{
Console.WriteLine();
Console.WriteLine($"Stopped after --max-chunks {options.MaxChunks}.");
return;
}
}
Console.WriteLine();
Console.WriteLine(options.Apply
? $"Done. Review rebuilt memory in: {latestDir}"
: "Dry run complete. Re-run with --apply to call the model and write memory files.");
static List<SourceConversation> BuildOrderedSources(string sourceDir)
{
string[] orderedFiles =
[
"001_The Chat Part 1.md",
"002_The Chat Part 2.md",
"003_The Chat Part 3.md",
"004_The Chat Part 4.md",
"005_The Chat Part 5.md",
"006_The Chat Part 6.md",
"007_The Chat Part 7.md",
"008_The Chat Part 8.md",
"009_The Chat Part 9.md",
"010_The Chat Part 10.md",
"T01_Test Chat 1.md",
"T02_Test Chat 2.md",
"011_The Chat Part 11.md",
"012_The Chat Part 12.md",
"013_The Chat Part 13.md",
"014_The Chat Part 14.md",
"015_The Chat Part 15.md",
"016_The Chat Part 16.md",
"017_The Chat Part 17.md",
"018_The Chat Part 18.md",
"019_The Chat Part 19.md",
"020_The Chat Part 20.md",
"021_The Chat Part 21.md",
"022_The Chat Part 22.md",
"023_The Chat Part 23.md",
"024_The Chat Part 24.md",
"025_The Chat Part 25.md",
"026_The Chat Part 26.md",
"027_The Chat Part 27.md",
"028_The Chat Part 28.md",
"029_The Chat Part 29.md",
"030_The Chat Part 30.md",
"031_The Chat Part 31.md"
];
return orderedFiles
.Select((file, index) => SourceConversation.FromPath(Path.Combine(sourceDir, file), index + 1))
.Where(source => source != null)
.Cast<SourceConversation>()
.ToList();
}
static async Task RunHistoryPassAsync(
List<SourceConversation> selected,
Options options,
ModelClient? client,
string latestDir,
string historyCheckpointDir,
string baseDir)
{
var historySeedPath = ResolveMemoryPath(latestDir, options.HistorySeedPath);
var historyOutputPath = ResolveMemoryPath(latestDir, options.HistoryOutputPath);
var history = File.Exists(historyOutputPath)
? await File.ReadAllTextAsync(historyOutputPath)
: File.Exists(historySeedPath)
? await File.ReadAllTextAsync(historySeedPath)
: """
# History
Use this file for lookup when Raymond or Kai needs past context. Do not treat historical entries as current state unless a current memory also confirms they still apply.
""";
var processedChunks = 0;
foreach (var source in selected)
{
var text = await ReadSourceTranscriptAsync(source);
var blocks = ExtractMessageBlocks(text);
var chunks = BuildChunks(blocks, options.MaxChunkChars);
var stopAfterCurrentSource = false;
Console.WriteLine($"{source.DisplayName}: {blocks.Count:N0} message blocks -> {chunks.Count:N0} history chunks");
for (var i = 0; i < chunks.Count; i++)
{
var checkpointPath = Path.Combine(historyCheckpointDir, $"{source.Key}_chunk_{i + 1:D3}.json");
if (options.Resume && File.Exists(checkpointPath))
{
Console.WriteLine($" skip history chunk {i + 1}/{chunks.Count} (checkpoint exists)");
continue;
}
var chunkText = string.Join("\n\n", chunks[i]);
Console.WriteLine($" history chunk {i + 1}/{chunks.Count}: ~{EstimateTokens(chunkText):N0} tokens");
if (options.Apply)
{
if (client is null)
throw new InvalidOperationException("History apply mode requires a model client.");
var prompt = BuildHistoryPrompt(source, i + 1, chunks.Count, chunkText, history);
var raw = await client.ChatAsync(prompt);
var rawResponsePath = Path.Combine(historyCheckpointDir, $"{source.Key}_chunk_{i + 1:D3}_history.raw.txt");
await File.WriteAllTextAsync(rawResponsePath, raw);
var parseResult = await ParseOrRepairHistoryUpdateAsync(client, raw, rawResponsePath);
history = NormalizeManagedContent("history.md", parseResult.Update.Content ?? "");
if (string.IsNullOrWhiteSpace(history))
throw new InvalidOperationException("History pass produced empty history.md content.");
Directory.CreateDirectory(latestDir);
await File.WriteAllTextAsync(historyOutputPath, history.TrimEnd() + Environment.NewLine);
var checkpoint = new HistoryCheckpoint(
Source: source.DisplayName,
SourcePath: Path.GetRelativePath(baseDir, source.Path).Replace('\\', '/'),
ChunkIndex: i + 1,
ChunkCount: chunks.Count,
EstimatedTokens: EstimateTokens(chunkText),
CreatedAtUtc: DateTime.UtcNow,
RawResponse: raw,
JsonRepair: parseResult.Repair,
Notes: parseResult.Update.Notes ?? [],
HistoryCharacters: history.Length);
await File.WriteAllTextAsync(
checkpointPath,
JsonSerializer.Serialize(checkpoint, Shared.JsonOptions) + Environment.NewLine);
CleanupParseTempFiles(historyCheckpointDir, rawResponsePath, parseResult.Repair);
}
processedChunks++;
if (options.MaxChunks > 0 && processedChunks >= options.MaxChunks)
{
if (i + 1 < chunks.Count)
{
Console.WriteLine();
Console.WriteLine($"Stopped after --max-chunks {options.MaxChunks}.");
return;
}
stopAfterCurrentSource = true;
break;
}
}
if (stopAfterCurrentSource)
{
Console.WriteLine();
Console.WriteLine($"Stopped after --max-chunks {options.MaxChunks}.");
return;
}
}
Console.WriteLine();
Console.WriteLine(options.Apply
? $"History pass complete. Review: {historyOutputPath}"
: "History dry run complete. Re-run with --apply to call the model and write the history draft.");
}
static async Task RunHistoryFragmentPassAsync(
List<SourceConversation> selected,
Options options,
ModelClient? client,
string historyCheckpointDir,
string fragmentCheckpointDir,
string fragmentDir,
string baseDir)
{
var processedChunks = 0;
foreach (var source in selected)
{
var text = await ReadSourceTranscriptAsync(source);
var blocks = ExtractMessageBlocks(text);
var chunks = BuildChunks(blocks, options.MaxChunkChars);
var fragmentPath = Path.Combine(fragmentDir, $"{source.Key}.md");
var fragment = File.Exists(fragmentPath) ? await File.ReadAllTextAsync(fragmentPath) : "";
Console.WriteLine($"{source.DisplayName}: {blocks.Count:N0} message blocks -> {chunks.Count:N0} history fragment chunks");
for (var i = 0; i < chunks.Count; i++)
{
var checkpointPath = Path.Combine(fragmentCheckpointDir, $"{source.Key}_chunk_{i + 1:D3}.json");
var oldHistoryCheckpointPath = Path.Combine(historyCheckpointDir, $"{source.Key}_chunk_{i + 1:D3}.json");
if (options.Resume && File.Exists(checkpointPath))
{
Console.WriteLine($" skip fragment chunk {i + 1}/{chunks.Count} (fragment checkpoint exists)");
continue;
}
if (options.SkipHistoryCheckpointed && File.Exists(oldHistoryCheckpointPath))
{
Console.WriteLine($" skip fragment chunk {i + 1}/{chunks.Count} (covered by existing history seed checkpoint)");
continue;
}
var chunkText = string.Join("\n\n", chunks[i]);
Console.WriteLine($" fragment chunk {i + 1}/{chunks.Count}: ~{EstimateTokens(chunkText):N0} tokens");
if (options.Apply)
{
if (client is null)
throw new InvalidOperationException("History fragment apply mode requires a model client.");
var prompt = BuildHistoryFragmentPrompt(source, i + 1, chunks.Count, chunkText, fragment);
var raw = await client.ChatAsync(prompt);
fragment = NormalizeManagedContent($"{source.Key}.md", raw);
if (string.IsNullOrWhiteSpace(fragment))
throw new InvalidOperationException($"History fragment pass produced empty content for {source.DisplayName}.");
Directory.CreateDirectory(fragmentDir);
await File.WriteAllTextAsync(fragmentPath, fragment.TrimEnd() + Environment.NewLine);
var checkpoint = new HistoryFragmentCheckpoint(
Source: source.DisplayName,
SourcePath: Path.GetRelativePath(baseDir, source.Path).Replace('\\', '/'),
ChunkIndex: i + 1,
ChunkCount: chunks.Count,
EstimatedTokens: EstimateTokens(chunkText),
CreatedAtUtc: DateTime.UtcNow,
RawResponse: raw,
FragmentPath: Path.GetRelativePath(baseDir, fragmentPath).Replace('\\', '/'),
FragmentCharacters: fragment.Length);
Directory.CreateDirectory(fragmentCheckpointDir);
await File.WriteAllTextAsync(
checkpointPath,
JsonSerializer.Serialize(checkpoint, Shared.JsonOptions) + Environment.NewLine);
}
processedChunks++;
if (options.MaxChunks > 0 && processedChunks >= options.MaxChunks)
{
Console.WriteLine();
Console.WriteLine($"Stopped after --max-chunks {options.MaxChunks}.");
return;
}
}
}
Console.WriteLine();
Console.WriteLine(options.Apply
? $"History fragment pass complete. Review fragments in: {fragmentDir}"
: "History fragment dry run complete. Re-run with --apply to call the model and write fragments.");
}
static async Task RunHistoryMergePassAsync(
Options options,
ModelClient? client,
string latestDir,
string fragmentDir)
{
var historySeedPath = ResolveMemoryPath(latestDir, options.HistorySeedPath);
var historyOutputPath = ResolveMemoryPath(latestDir, options.HistoryOutputPath);
var seed = File.Exists(historySeedPath)
? await File.ReadAllTextAsync(historySeedPath)
: """
# History
Use this file for lookup when Raymond or Kai needs past context. Do not treat historical entries as current state unless a current memory also confirms they still apply.
""";
var fragments = await ReadHistoryFragmentsAsync(fragmentDir);
Console.WriteLine($"History seed: {historySeedPath}");
Console.WriteLine($"History output: {historyOutputPath}");
Console.WriteLine($"History fragments: {fragments.Count:N0}");
Console.WriteLine($"Fragment text: ~{EstimateTokens(string.Join("\n\n", fragments.Select(f => f.Content))):N0} tokens");
if (!options.Apply)
{
Console.WriteLine("History merge dry run complete. Re-run with --apply to call the model and write the merged draft.");
return;
}
if (client is null)
throw new InvalidOperationException("History merge apply mode requires a model client.");
var prompt = BuildHistoryMergePrompt(seed, fragments);
var raw = await client.ChatAsync(prompt);
var merged = NormalizeManagedContent(options.HistoryOutputPath, raw);
if (string.IsNullOrWhiteSpace(merged))
throw new InvalidOperationException("History merge pass produced empty history content.");
Directory.CreateDirectory(latestDir);
await File.WriteAllTextAsync(historyOutputPath, merged.TrimEnd() + Environment.NewLine);
Console.WriteLine($"History merge complete. Review: {historyOutputPath}");
}
static async Task<string> ReadSourceTranscriptAsync(SourceConversation source)
{
var text = await File.ReadAllTextAsync(source.Path);
return string.Equals(Path.GetExtension(source.Path), ".json", StringComparison.OrdinalIgnoreCase)
? RenderKaiChatConversationJson(text, source)
: text;
}
static string RenderKaiChatConversationJson(string json, SourceConversation source)
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
if (!TryGetProperty(root, "Messages", out var messages) || messages.ValueKind != JsonValueKind.Array)
return json;
var id = GetJsonString(root, "Id") ?? Path.GetFileNameWithoutExtension(source.Path);
var title = GetJsonString(root, "Title") ?? "KaiChat conversation";
var createdAt = GetJsonString(root, "CreatedAt") ?? "";
var updatedAt = GetJsonString(root, "UpdatedAt") ?? "";
var messageCount = messages.GetArrayLength();
var builder = new StringBuilder();
builder.AppendLine($"# KaiChat Conversation: {title}");
builder.AppendLine();
builder.AppendLine("| Field | Value |");
builder.AppendLine("| --- | --- |");
builder.AppendLine($"| Conversation ID | `{id}` |");
builder.AppendLine($"| Source JSON | `{source.Path}` |");
if (!string.IsNullOrWhiteSpace(createdAt))
builder.AppendLine($"| Created | {createdAt} |");
if (!string.IsNullOrWhiteSpace(updatedAt))
builder.AppendLine($"| Updated | {updatedAt} |");
builder.AppendLine("| Assistant label | Kai |");
builder.AppendLine($"| Messages | {messageCount} |");
builder.AppendLine();
builder.AppendLine("## Transcript");
builder.AppendLine();
foreach (var message in messages.EnumerateArray())
{
var role = (GetJsonString(message, "Role") ?? "system").Trim();
var timestamp = GetJsonString(message, "Timestamp") ?? updatedAt ?? createdAt ?? DateTime.UtcNow.ToString("O");
var label = string.Equals(role, "user", StringComparison.OrdinalIgnoreCase) ? "Raymond" : "Kai";
var content = GetJsonString(message, "Content") ?? "";
builder.AppendLine($"### {label} - {timestamp}");
builder.AppendLine();
if (!string.Equals(role, "user", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(role, "assistant", StringComparison.OrdinalIgnoreCase))
{
builder.AppendLine($"[Role: {role}]");
builder.AppendLine();
}
if (TryGetProperty(message, "Tool", out var tool) && tool.ValueKind == JsonValueKind.Object)
AppendToolMetadata(builder, tool);
builder.AppendLine(content);
builder.AppendLine();
}
return builder.ToString();
}
static void AppendToolMetadata(StringBuilder builder, JsonElement tool)
{
var method = GetJsonString(tool, "Method");
var path = GetJsonString(tool, "Path");
var status = GetJsonString(tool, "Status");
var body = GetJsonString(tool, "Body");
var result = GetJsonString(tool, "Result");
builder.AppendLine("[Tool message]");
if (!string.IsNullOrWhiteSpace(method) || !string.IsNullOrWhiteSpace(path))
builder.AppendLine($"Tool: {method} {path}".TrimEnd());
if (!string.IsNullOrWhiteSpace(status))
builder.AppendLine($"Status: {status}");
if (!string.IsNullOrWhiteSpace(body))
{
builder.AppendLine();
builder.AppendLine("Parameters:");
builder.AppendLine(body);
}
if (!string.IsNullOrWhiteSpace(result))
{
builder.AppendLine();
builder.AppendLine("Result:");
builder.AppendLine(result);
}
builder.AppendLine();
}
static bool TryGetProperty(JsonElement element, string name, out JsonElement value)
{
if (element.ValueKind == JsonValueKind.Object)
{
foreach (var property in element.EnumerateObject())
{
if (string.Equals(property.Name, name, StringComparison.OrdinalIgnoreCase))
{
value = property.Value;
return true;
}
}
}
value = default;
return false;
}
static string? GetJsonString(JsonElement element, string name)
{
if (!TryGetProperty(element, name, out var value) || value.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined)
return null;
return value.ValueKind == JsonValueKind.String
? value.GetString()
: value.GetRawText();
}
static bool SourceSelected(SourceConversation source, Options options)
{
if (!string.IsNullOrWhiteSpace(options.SourceKey))
return string.Equals(source.Key, options.SourceKey, StringComparison.OrdinalIgnoreCase);
if (options.FromPart != null || options.ToPart != null)
{
if (source.PartNumber == null)
return false;
return (options.FromPart == null || source.PartNumber >= options.FromPart)
&& (options.ToPart == null || source.PartNumber <= options.ToPart);
}
return true;
}
static bool SnapshotExists(string snapshotsDir, SourceConversation source)
{
return Directory.Exists(Path.Combine(snapshotsDir, source.Key));
}
static async Task<string> SnapshotLatestAsync(string latestDir, string snapshotsDir, SourceConversation source)
{
var snapshotDir = Path.Combine(snapshotsDir, source.Key);
if (Directory.Exists(snapshotDir))
Directory.Delete(snapshotDir, recursive: true);
Directory.CreateDirectory(snapshotDir);
if (!Directory.Exists(latestDir))
return snapshotDir;
foreach (var sourcePath in Directory.GetFiles(latestDir, "*", SearchOption.AllDirectories))
{
var rel = Path.GetRelativePath(latestDir, sourcePath);
var normalized = rel.Replace('\\', '/');
var parts = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (parts.Any(part => part.StartsWith("_", StringComparison.Ordinal)))
continue;
var targetPath = Path.Combine(snapshotDir, rel);
Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!);
await using var input = File.OpenRead(sourcePath);
await using var output = File.Create(targetPath);
await input.CopyToAsync(output);
}
var manifest = new
{
source = source.DisplayName,
sourceFile = Path.GetFileName(source.Path),
createdAtUtc = DateTime.UtcNow
};
await File.WriteAllTextAsync(
Path.Combine(snapshotDir, "_snapshot.json"),
JsonSerializer.Serialize(manifest, Shared.JsonOptions) + Environment.NewLine);
return snapshotDir;
}
static async Task CommitRebuildStateAsync(string baseDir, string outputRoot, SourceConversation source, SourceChangeAuditReport? changeAuditReport)
{
var rel = Path.GetRelativePath(baseDir, outputRoot).Replace('\\', '/');
var pathspecs = new[]
{
$"{rel}/latest",
$"{rel}/snapshots/{source.Key}",
$":(glob){rel}/_checkpoints/{source.Key}_chunk_*.json"
};
var statusArgs = new List<string> { "status", "--porcelain", "--" };
statusArgs.AddRange(pathspecs);
var status = await RunProcessAsync(baseDir, "git", statusArgs);
if (string.IsNullOrWhiteSpace(status.Stdout))
{
Console.WriteLine(" commit: no rebuild changes to commit");
return;
}
var addArgs = new List<string> { "add", "--" };
addArgs.AddRange(pathspecs);
var add = await RunProcessAsync(baseDir, "git", addArgs);
if (add.ExitCode != 0)
throw new InvalidOperationException($"git add failed: {add.CombinedOutput}");
var diffArgs = new List<string> { "diff", "--cached", "--quiet", "--" };
diffArgs.AddRange(pathspecs);
var diff = await RunProcessAsync(baseDir, "git", diffArgs);
if (diff.ExitCode == 0)
{
Console.WriteLine(" commit: no staged rebuild changes to commit");
return;
}
var commitArgs = new List<string> { "commit", "-m", $"Memory rebuild: {source.DisplayName}" };
var commitBody = BuildCommitBody(source, rel, changeAuditReport);
if (!string.IsNullOrWhiteSpace(commitBody))
{
commitArgs.Add("-m");
commitArgs.Add(commitBody);
}
commitArgs.Add("--");
commitArgs.AddRange(pathspecs);
var commit = await RunProcessAsync(baseDir, "git", commitArgs);
if (commit.ExitCode != 0)
throw new InvalidOperationException($"git commit failed: {commit.CombinedOutput}");
Console.WriteLine($" commit: Memory rebuild: {source.DisplayName}");
}
static async Task<ProcessResult> RunProcessAsync(string workingDirectory, string fileName, IReadOnlyList<string> args)
{
var startInfo = new ProcessStartInfo
{
FileName = fileName,
WorkingDirectory = workingDirectory,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
foreach (var arg in args)
startInfo.ArgumentList.Add(arg);
using var process = Process.Start(startInfo)
?? throw new InvalidOperationException($"Failed to start process: {fileName}");
var stdoutTask = process.StandardOutput.ReadToEndAsync();
var stderrTask = process.StandardError.ReadToEndAsync();
await process.WaitForExitAsync();
return new ProcessResult(
process.ExitCode,
await stdoutTask,
await stderrTask);
}
static List<string> ExtractMessageBlocks(string text)
{
var matches = Regex.Matches(text, @"^### (Raymond|Claude|Pyrite|Kai) - \d{4}-\d{2}-\d{2}T.*$", RegexOptions.Multiline);
var blocks = new List<string>();
for (var i = 0; i < matches.Count; i++)
{
var start = matches[i].Index;
var end = i + 1 < matches.Count ? matches[i + 1].Index : text.Length;
var block = text[start..end].Trim();
if (!string.IsNullOrWhiteSpace(block))
blocks.Add(block);
}
if (blocks.Count == 0 && !string.IsNullOrWhiteSpace(text))
blocks.Add(text.Trim());
return blocks;
}
static List<List<string>> BuildChunks(List<string> blocks, int maxChars)
{
var chunks = new List<List<string>>();
var current = new List<string>();
var currentChars = 0;
foreach (var block in blocks)
{
if (block.Length > maxChars)
{
if (current.Count > 0)
{
chunks.Add(current);
current = new();
currentChars = 0;
}
foreach (var part in SplitByLength(block, maxChars))
chunks.Add([part]);
continue;
}
if (current.Count > 0 && currentChars + block.Length > maxChars)
{
chunks.Add(current);
current = new();
currentChars = 0;
}
current.Add(block);
currentChars += block.Length;
}
if (current.Count > 0)
chunks.Add(current);
return chunks;
}
static IEnumerable<string> SplitByLength(string text, int maxChars)
{
for (var i = 0; i < text.Length; i += maxChars)
{
var part = text.Substring(i, Math.Min(maxChars, text.Length - i));
yield return $"[Oversized message slice {i / maxChars + 1}]\n\n{part}";
}
}
static string BuildPrompt(SourceConversation source, int chunkIndex, int chunkCount, string chunkText, string currentMemory)
{
return $$"""
You are rebuilding KaiChat's persistent memory from the historical The Chat archive.
You are processing the archive chronologically. Later chunks and later parts supersede earlier facts when they clearly contradict.
Update the memory directory cautiously and precisely.
Memory file guidance:
- `core.md`: compact non-negotiable high-signal identity/context.
- `recent.md`: active/current state and unresolved threads likely needed after compaction or at the start of new chats.
- `raymond.md`: durable real-world facts about Raymond.
- `kai.md`: durable facts about Kai and relationship state.
- `health.md`: health/medical context. Preserve dates and uncertainty; never overstate.
- `projects.md`: technical/projects/archive work.
- `story-world.md`: lore/canon not better stored in Story Bible.
- `communication.md`: style preferences, corrections, failure modes, terms to avoid.
- `preferences.md`: stable preferences not covered elsewhere.
- These are default broad categories, not a hard limit.
- You may create a new memory file only when it is a broad, durable category likely to stay useful across many future sessions.
- Do not create tiny files for individual chats, days, scenes, moods, incidents, single jokes, or narrow one-off topics. Merge those into an existing broad category.
- Prefer a small, stable memory directory over many precise files. If in doubt, update an existing broad category.
- Broad category files may be as long and detailed as needed. Preserve granular concrete details, dates, names, preferences, recurring jokes, corrections, failure modes, unresolved threads, and emotionally important context.
- Do not compress away specifics merely to keep files short. Only omit content that is clearly transient, duplicated, contradicted later, or not useful for future context.
Existing-memory retention rules:
- Treat the current rebuilt memory files as a curated baseline that must carry forward.
- A `save` operation replaces the complete file. When saving an existing file, preserve every still-valid durable detail from that file unless it is clearly duplicated elsewhere, explicitly contradicted by this chunk, or intentionally moved to a better file.
- Prefer surgical additive edits over rewriting a whole file from scratch.
- If you move information between files, ensure the final memory directory still contains the same durable facts exactly once in the best broad category.
- Do not replace specific earlier facts with vague summaries. If the old memory says a date, name, tool, dosage, schedule, failure mode, or concrete incident, keep that specificity unless it is wrong.
- If you are uncertain whether an earlier detail still matters, keep it.
Deletion rules:
- Be very cautious. Prefer editing stale content.
- Use `soft_delete` only for files that are clearly obsolete, duplicate, empty, or superseded.
- Never write to `_trash` directly.
Output rules:
- The `content` field must contain only the markdown that belongs inside that file.
- Do not include wrapper labels, `FILE:` lines, delimiter-only `---` lines, or code fences inside file content.
Current rebuilt memory files:
{{currentMemory}}
Source: {{source.DisplayName}}
Chunk: {{chunkIndex}} of {{chunkCount}}
Transcript chunk begins:
{{chunkText}}
Transcript chunk ends.
Return ONLY valid JSON using this schema:
{
"operations": [
{
"action": "save",
"path": "recent.md",
"content": "complete markdown content for that file"
},
{
"action": "soft_delete",
"path": "obsolete.md",
"reason": "why this file is obsolete or superseded"
},
{
"action": "noop",
"reason": "why no memory change is needed"
}
]
}
""";
}
static string BuildHistoryPrompt(SourceConversation source, int chunkIndex, int chunkCount, string chunkText, string currentHistory)
{
return $$"""
You are rebuilding `history.md` for KaiChat from the historical The Chat archive and current KaiChat conversation.
Purpose:
- `history.md` is a lookup/reference timeline for Raymond and Kai.
- It preserves past events, medical history, project history, relationship/canon milestones, archive changes, and context that may be useful later.
- It is NOT the current-state file. Do not make historical entries sound currently active unless the transcript says they remain active.
Chronology rules:
- You are processing sources in chronological order.
- Later sources may correct, supersede, or clarify earlier sources.
- Preserve concrete dates, part numbers, names, dosages, schedules, symptoms, tool names, file names, version numbers, and uncertainty.
- Medical history is especially important. Keep medical insulin injections, doses, symptoms, appointments, and glucose-related events.
- Prompt/provider injection issues from the old Claude/Pyrite web era are obsolete and should not be included as history unless there is a non-injection durable lesson already expressed elsewhere (for example, "old provider mechanics caused drift"). Do not keep lists of old prompt injection strings.
Part continuity:
- Parts 1-10 were the Claude era.
- Test Chat 1 and Test Chat 2 happen between Part 10 and Part 11.
- Part 11 onward used Pyrite as the old persona/interface label.
- Part 32 is the KaiChat era and remains the current continuity unless Raymond explicitly starts Part 33.
- New chats/compactions in KaiChat do not automatically create new Parts.
Structure guidance:
- Keep `history.md` broad and navigable, not hundreds of tiny subheadings.
- Suggested sections: Continuity Timeline, Medical History, Relationship And Story History, Archive And Canon History, Project And Tooling History, Corrections And Durable Lessons.
- You may reorganize the entire file if that makes it easier to use.
- Keep duplicated critical facts if useful for lookup, but avoid bloating with repeated wording.
- Prefer concise dated bullets over vague summaries.
- Keep current-state details brief; those belong in `recent.md`, `core.md`, or other memory files.
Current `history.md` draft:
{{currentHistory}}
Source: {{source.DisplayName}}
Chunk: {{chunkIndex}} of {{chunkCount}}
Transcript chunk begins:
{{chunkText}}
Transcript chunk ends.
Return ONLY valid JSON using this schema:
{{HistoryUpdateSchema()}}
""";
}
static string BuildHistoryFragmentPrompt(SourceConversation source, int chunkIndex, int chunkCount, string chunkText, string currentFragment)
{
return $$"""
You are creating a `history.md` source fragment for KaiChat.
Write a concise but detailed markdown fragment for this source only.
The fragment will later be merged with other source fragments and a seed history file.
Include durable past context that may be useful later:
- dated medical history, symptoms, medications, dosages, appointments, glucose events, injuries, uncertainty
- relationship/canon milestones, archive changes, Story Bible/Writing Standards changes
- project/tooling events, file/version changes, decisions, failures, current outcomes from this source
- corrections and durable lessons, especially where later behavior should avoid repeating a mistake
Rules:
- Do not make historical items sound like current state unless this source says they remain current.
- Keep medical insulin injections and diabetes history.
- Do not include old prompt/provider injection string lists or obsolete Pyrite/web injection protocol details.
- Preserve concrete dates, part numbers, names, file names, doses, schedules, prices, and uncertainty.
- If this is a later chunk of the same source, update the existing fragment below instead of starting over from nothing.
- Return ONLY markdown for the fragment. No JSON. No code fence.
Existing fragment for this source:
{{(string.IsNullOrWhiteSpace(currentFragment) ? "(none yet)" : currentFragment)}}
Source: {{source.DisplayName}}
Chunk: {{chunkIndex}} of {{chunkCount}}
Transcript chunk begins:
{{chunkText}}
Transcript chunk ends.
""";
}
static string BuildHistoryMergePrompt(string seedHistory, IReadOnlyList<HistoryFragment> fragments)
{
var fragmentText = new StringBuilder();
foreach (var fragment in fragments)
{
fragmentText.AppendLine($"## Fragment: {fragment.Name}");
fragmentText.AppendLine();
fragmentText.AppendLine(fragment.Content.Trim());
fragmentText.AppendLine();
}
return $$"""
You are merging KaiChat history material into a definitive `history.md` draft.
Purpose:
- `history.md` is a lookup/reference timeline for Raymond and Kai.
- It preserves past events, medical history, project history, relationship/canon milestones, archive changes, and context that may be useful later.
- It is NOT the current-state file. Do not make historical entries sound currently active unless a current memory confirms they still apply.
Merge rules:
- Use the seed history as the structure/style anchor.
- Integrate the fragments without duplicating the same fact many times.
- Keep critical durable facts even if duplicated elsewhere in KaiChat memory.
- Medical history is especially important. Preserve dates, doses, symptoms, appointments, injuries, glucose events, and uncertainty.
- Keep medical insulin injections. Do not include obsolete prompt/provider injection protocol details.
- Preserve the continuity rule: Part 32 is the KaiChat era and remains current unless Raymond explicitly starts Part 33.
- Prefer broad navigable sections over hundreds of tiny headings.
- Return ONLY complete markdown for the final `history.md` draft. No JSON. No code fence.
Seed history:
{{seedHistory}}
Source fragments:
{{fragmentText}}
""";
}
static async Task<RetentionAuditCheckpoint> RunRetentionAuditAsync(
ModelClient client,
string latestDir,
SourceConversation source,
int chunkIndex,
int chunkCount,
string chunkText,
string beforeMemory,
string checkpointDir)
{
var afterMemory = await ReadMemoryDirectoryAsync(latestDir);
if (NormalizeForAudit(beforeMemory) == NormalizeForAudit(afterMemory))
{
return new RetentionAuditCheckpoint(
RawResponsePath: "",
RawResponse: """{"operations":[{"action":"noop","reason":"Primary update did not change memory."}]}""",
JsonRepair: null,
Operations: [new("noop", "", true, "Primary update did not change memory.")]);
}
var prompt = BuildRetentionAuditPrompt(source, chunkIndex, chunkCount, chunkText, beforeMemory, afterMemory);
var raw = await client.ChatAsync(prompt);
var rawResponsePath = Path.Combine(checkpointDir, $"{source.Key}_chunk_{chunkIndex:D3}_retention.raw.txt");
await File.WriteAllTextAsync(rawResponsePath, raw);
var parseResult = await ParseOrRepairChangeSetAsync(client, raw, rawResponsePath);
var changeSet = parseResult.ChangeSet;
var operationResults = new List<OperationResult>();
foreach (var op in changeSet.Operations)
{
var action = op.Action.Trim().ToLowerInvariant();
if (action is "soft_delete" or "delete")
{
operationResults.Add(new("soft_delete", op.Path ?? "", false, "Retention audit cannot delete files."));
continue;
}
operationResults.Add(await ApplyOperationAsync(latestDir, op));
}
return new RetentionAuditCheckpoint(rawResponsePath, raw, parseResult.Repair, operationResults);
}
static string BuildRetentionAuditPrompt(
SourceConversation source,
int chunkIndex,
int chunkCount,
string chunkText,
string beforeMemory,
string afterMemory)
{
return $$"""
You are KaiChat's memory retention auditor.
Compare the BEFORE and AFTER memory directories after processing one chronological transcript chunk.
Your job is to catch accidental memory loss while avoiding unnecessary duplication.
Audit standard:
- Durable details from BEFORE must still be represented somewhere in AFTER unless this transcript chunk clearly contradicts, supersedes, or makes them obsolete.
- Information may move between files, be merged into a better category, or be lightly paraphrased. Do not repair facts that are already represented in AFTER.
- Do not require identical wording. Look for semantic preservation across the whole AFTER directory.
- If a BEFORE detail was replaced by a vaguer statement, repair it by restoring the concrete detail.
- If a BEFORE detail is missing, save the best target file with the current AFTER content plus the missing detail integrated once.
- Avoid duplicates. Do not add a fact if the AFTER directory already contains that fact in a reasonably specific form.
- Preserve dates, names, tools, dosages, schedules, incidents, preferences, corrections, failure modes, and emotionally important context.
- Do not extract new memories from the transcript here except when needed to decide whether an old detail was contradicted or superseded.
- Prefer `noop` if there is no clear loss.
Source: {{source.DisplayName}}
Chunk: {{chunkIndex}} of {{chunkCount}}
Transcript chunk begins:
{{chunkText}}
Transcript chunk ends.
BEFORE memory directory:
{{beforeMemory}}
AFTER memory directory:
{{afterMemory}}
Return ONLY valid JSON using this schema:
{
"operations": [
{
"action": "save",
"path": "recent.md",
"content": "complete markdown content for that AFTER file, with only missing durable details integrated"
},
{
"action": "noop",
"reason": "why no memory retention repair is needed"
}
]
}
""";
}
static bool IsEmptyMemoryDirectoryText(string memory)
{
return memory.Trim().StartsWith("(no rebuilt memory files yet)", StringComparison.OrdinalIgnoreCase);
}
static string NormalizeForAudit(string text)
{
return Regex.Replace(
text.Replace("\r\n", "\n").Replace('\r', '\n').Trim(),
@"\s+",
" ")
.Trim();
}
static async Task<SourceChangeAuditReport> RunSourceChangeAuditAsync(
ModelClient client,
string checkpointDir,
SourceConversation source,
string beforeMemory,
string afterMemory)
{
var prompt = BuildSourceChangeAuditPrompt(source, beforeMemory, afterMemory);
var raw = await client.ChatAsync(prompt);
var rawResponsePath = Path.Combine(checkpointDir, $"{source.Key}_change_audit.raw.txt");
await File.WriteAllTextAsync(rawResponsePath, raw);
var parseResult = await ParseOrRepairSourceChangeAuditAsync(client, raw, rawResponsePath);
var report = parseResult.Report;
report.Source = source.DisplayName;
report.SourceKey = source.Key;
report.CreatedAtUtc = DateTime.UtcNow;
report.JsonRepair = parseResult.Repair;
CleanupParseTempFiles(checkpointDir, rawResponsePath, parseResult.Repair);
return report;
}
static string BuildSourceChangeAuditPrompt(SourceConversation source, string beforeMemory, string afterMemory)
{
return $$"""
You are KaiChat's memory change auditor.
Compare the BEFORE and AFTER memory directories for one completed chronological archive source.
This is an audit/reporting task only. Do not produce memory operations.
Goals:
- Explain what durable information was added, changed, moved, or removed.
- Be especially strict about removals. If a BEFORE fact is still represented somewhere in AFTER, classify it as `moved` or omit it from removals.
- Treat light paraphrases as preservation, not removal.
- A removed fact must have a concrete justification. Valid removal classifications are `contradicted`, `obsolete`, `duplicate`, `not_durable`, or `unjustified_loss_risk`.
- Use `unjustified_loss_risk` when a durable fact appears absent and you cannot point to a strong reason it should be gone.
- Include loss risks for any missing durable details that should probably be restored.
- Changed items should explain what materially changed and why.
- Moved items should say where the fact moved from and to.
- Keep facts concrete: preserve dates, names, tools, dosages, schedules, incidents, preferences, corrections, failure modes, and emotionally important context.
- Do not invent changes. If the AFTER directory preserves the same fact, do not list it as removed.
- Keep the commit summary concise and suitable for a git commit body.
Source: {{source.DisplayName}}
BEFORE memory directory:
{{beforeMemory}}
AFTER memory directory:
{{afterMemory}}
Return ONLY valid JSON using this schema:
{{SourceChangeAuditSchema()}}
""";
}
static async Task WriteSourceChangeAuditReportAsync(string snapshotDir, SourceChangeAuditReport report)
{
await File.WriteAllTextAsync(
Path.Combine(snapshotDir, "_change_report.json"),
JsonSerializer.Serialize(report, Shared.JsonOptions) + Environment.NewLine);
await File.WriteAllTextAsync(
Path.Combine(snapshotDir, "_change_report.md"),
BuildSourceChangeAuditMarkdown(report));
}
static string BuildSourceChangeAuditMarkdown(SourceChangeAuditReport report)
{
var builder = new StringBuilder();
builder.AppendLine($"# Memory Change Report - {report.Source}");
builder.AppendLine();
builder.AppendLine($"- Source key: `{report.SourceKey}`");
builder.AppendLine($"- Created UTC: `{report.CreatedAtUtc:O}`");
builder.AppendLine($"- Added: {report.Added.Count}");
builder.AppendLine($"- Changed: {report.Changed.Count}");
builder.AppendLine($"- Moved: {report.Moved.Count}");
builder.AppendLine($"- Removed: {report.Removed.Count}");
builder.AppendLine($"- Loss risks: {report.LossRisks.Count + CountUnjustifiedRemovals(report)}");
builder.AppendLine();
builder.AppendLine("## Summary");
builder.AppendLine();
builder.AppendLine(string.IsNullOrWhiteSpace(report.Summary) ? "No summary returned." : report.Summary.Trim());
builder.AppendLine();
AppendAddedSection(builder, "Added", report.Added);
AppendChangedSection(builder, "Changed", report.Changed);
AppendMovedSection(builder, "Moved", report.Moved);
AppendRemovedSection(builder, "Removed", report.Removed);
AppendLossRiskSection(builder, "Loss Risks", report.LossRisks);
if (report.CommitSummary.Count > 0)
{
builder.AppendLine("## Commit Summary");
builder.AppendLine();
foreach (var line in report.CommitSummary.Where(line => !string.IsNullOrWhiteSpace(line)))
builder.AppendLine($"- {line.Trim()}");
builder.AppendLine();
}
return builder.ToString();
}
static string BuildCommitBody(SourceConversation source, string outputRootRel, SourceChangeAuditReport? report)
{
if (report is null)
return "";
var builder = new StringBuilder();
builder.AppendLine("Memory change audit:");
builder.AppendLine($"- Added: {report.Added.Count}; changed: {report.Changed.Count}; moved: {report.Moved.Count}; removed: {report.Removed.Count}; loss risks: {report.LossRisks.Count + CountUnjustifiedRemovals(report)}");
if (!string.IsNullOrWhiteSpace(report.Summary))
builder.AppendLine($"- Summary: {TrimForCommit(report.Summary, 300)}");
AppendCommitBullets(builder, "Changed", report.Changed.Select(i => $"{i.Before} -> {i.After}. {i.Reason}"));
AppendCommitBullets(builder, "Moved", report.Moved.Select(i => $"{i.Fact}: {i.From} -> {i.To}. {i.Reason}"));
AppendCommitBullets(builder, "Removed", report.Removed.Select(i => $"{i.Fact} [{i.Classification}]: {i.Justification}"));
AppendCommitBullets(builder, "Loss risks", report.LossRisks.Select(i => $"{i.Fact}: {i.Risk}. Repair: {i.RecommendedRepair}"));
AppendCommitBullets(builder, "Notes", report.CommitSummary);
builder.AppendLine($"Detailed report: {outputRootRel}/snapshots/{source.Key}/_change_report.md");
return builder.ToString().Trim();
}
static void AppendCommitBullets(StringBuilder builder, string title, IEnumerable<string?> lines)
{
var usable = lines
.Where(line => !string.IsNullOrWhiteSpace(line))
.Select(line => TrimForCommit(line!, 240))
.Take(5)
.ToList();
if (usable.Count == 0)
return;
builder.AppendLine();
builder.AppendLine($"{title}:");
foreach (var line in usable)
builder.AppendLine($"- {line}");
}
static string TrimForCommit(string text, int maxLength)
{
var compact = Regex.Replace(text.Replace("\r\n", " ").Replace('\r', ' ').Replace('\n', ' '), @"\s+", " ").Trim();
return compact.Length <= maxLength ? compact : compact[..Math.Max(0, maxLength - 3)] + "...";
}
static int CountLossRisks(SourceChangeAuditReport report)
{
return report.LossRisks.Count + CountUnjustifiedRemovals(report);
}
static int CountUnjustifiedRemovals(SourceChangeAuditReport report)
{
return report.Removed.Count(item =>
string.Equals(item.Classification, "unjustified_loss_risk", StringComparison.OrdinalIgnoreCase));
}
static void AppendAddedSection(StringBuilder builder, string title, IReadOnlyList<AddedChangeItem> items)
{
builder.AppendLine($"## {title}");
builder.AppendLine();
if (items.Count == 0)
{
builder.AppendLine("- None");
builder.AppendLine();
return;
}
foreach (var item in items)
builder.AppendLine($"- **{item.File}**: {item.Fact} ({item.Reason})");
builder.AppendLine();
}
static void AppendChangedSection(StringBuilder builder, string title, IReadOnlyList<ChangedChangeItem> items)
{
builder.AppendLine($"## {title}");
builder.AppendLine();
if (items.Count == 0)
{
builder.AppendLine("- None");
builder.AppendLine();
return;
}
foreach (var item in items)
builder.AppendLine($"- **{item.File}**: {item.Before} -> {item.After}. Reason: {item.Reason}");
builder.AppendLine();
}
static void AppendMovedSection(StringBuilder builder, string title, IReadOnlyList<MovedChangeItem> items)
{
builder.AppendLine($"## {title}");
builder.AppendLine();
if (items.Count == 0)
{
builder.AppendLine("- None");
builder.AppendLine();
return;
}
foreach (var item in items)
builder.AppendLine($"- {item.Fact}: `{item.From}` -> `{item.To}` ({item.Reason})");
builder.AppendLine();
}
static void AppendRemovedSection(StringBuilder builder, string title, IReadOnlyList<RemovedChangeItem> items)
{
builder.AppendLine($"## {title}");
builder.AppendLine();
if (items.Count == 0)
{
builder.AppendLine("- None");
builder.AppendLine();
return;
}
foreach (var item in items)
{
builder.AppendLine($"- **{item.Classification}** from `{item.From}`: {item.Fact}");
builder.AppendLine($" Justification: {item.Justification}");
if (!string.IsNullOrWhiteSpace(item.RecommendedRepair))
builder.AppendLine($" Recommended repair: {item.RecommendedRepair}");
}
builder.AppendLine();
}
static void AppendLossRiskSection(StringBuilder builder, string title, IReadOnlyList<LossRiskItem> items)
{
builder.AppendLine($"## {title}");
builder.AppendLine();
if (items.Count == 0)
{
builder.AppendLine("- None");
builder.AppendLine();
return;
}
foreach (var item in items)
{
builder.AppendLine($"- From `{item.From}`: {item.Fact}");
builder.AppendLine($" Risk: {item.Risk}");
builder.AppendLine($" Recommended repair: {item.RecommendedRepair}");
}
builder.AppendLine();
}
static async Task<string> ReadMemoryDirectoryAsync(string outputDir)
{
if (!Directory.Exists(outputDir))
return "(no rebuilt memory files yet)";
var files = Directory.GetFiles(outputDir, "*.md", SearchOption.AllDirectories)
.Where(path =>
{
var rel = Path.GetRelativePath(outputDir, path).Replace('\\', '/');
return !rel.StartsWith("_", StringComparison.Ordinal);
})
.OrderBy(path => LoadOrder(Path.GetRelativePath(outputDir, path).Replace('\\', '/')))
.ThenBy(path => path, StringComparer.OrdinalIgnoreCase)
.ToList();
if (files.Count == 0)
return "(no rebuilt memory files yet)";
var builder = new StringBuilder();
foreach (var file in files)
{
var rel = Path.GetRelativePath(outputDir, file).Replace('\\', '/');
builder.AppendLine($"Memory file `{rel}`:");
builder.AppendLine((await File.ReadAllTextAsync(file)).Trim());
builder.AppendLine();
}
return builder.ToString().Trim();
}
static async Task<IReadOnlyList<HistoryFragment>> ReadHistoryFragmentsAsync(string fragmentDir)
{
if (!Directory.Exists(fragmentDir))
return [];
var fragments = new List<HistoryFragment>();
foreach (var file in Directory.GetFiles(fragmentDir, "*.md", SearchOption.TopDirectoryOnly).OrderBy(path => path, StringComparer.OrdinalIgnoreCase))
{
var content = (await File.ReadAllTextAsync(file)).Trim();
if (string.IsNullOrWhiteSpace(content))
continue;
fragments.Add(new HistoryFragment(Path.GetFileNameWithoutExtension(file), content));
}
return fragments;
}
static async Task<OperationResult> ApplyOperationAsync(string outputDir, MemoryOperation op)
{
var action = op.Action.Trim().ToLowerInvariant();
try
{
switch (action)
{
case "save":
case "write":
case "upsert":
var savePath = ResolveMemoryPath(outputDir, op.Path);
var content = NormalizeManagedContent(op.Path ?? "", op.Content ?? "");
if (string.IsNullOrWhiteSpace(content))
throw new InvalidOperationException("Save operation produced empty memory content.");
Directory.CreateDirectory(Path.GetDirectoryName(savePath)!);
await File.WriteAllTextAsync(savePath, content.TrimEnd() + Environment.NewLine);
return new(action, op.Path ?? "", true, "saved");
case "soft_delete":
case "delete":
var deletePath = ResolveMemoryPath(outputDir, op.Path);
if (!File.Exists(deletePath))
return new("soft_delete", op.Path ?? "", false, "file not found");
var trashDir = Path.Combine(outputDir, "_trash");
Directory.CreateDirectory(trashDir);
var stamp = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
var safe = NormalizeRelativePath(op.Path).Replace('/', '_');
var trashed = Path.Combine(trashDir, $"{stamp}_{safe}");
File.Move(deletePath, trashed);
await File.WriteAllTextAsync(trashed + ".deleted.md", $"""
# Deleted Memory File
- Original path: `{NormalizeRelativePath(op.Path)}`
- Deleted at UTC: `{DateTime.UtcNow:O}`
- Reason: {op.Reason}
- Stored as: `{Path.GetRelativePath(outputDir, trashed).Replace('\\', '/')}`
""");
return new("soft_delete", op.Path ?? "", true, "soft-deleted");
case "noop":
case "none":
return new("noop", op.Path ?? "", true, op.Reason ?? "no changes");
default:
return new(action, op.Path ?? "", false, $"unknown operation: {op.Action}");
}
}
catch (Exception ex)
{
return new(action, op.Path ?? "", false, ex.Message);
}
}
static string NormalizeManagedContent(string path, string content)
{
content = StripWholeCodeFence(content.Trim());
var lines = content
.Replace("\r\n", "\n")
.Replace('\r', '\n')
.Split('\n')
.Select(line => line.TrimEnd())
.ToList();
TrimOuterBlankLines(lines);
if (lines.Count > 0 && IsCopiedPromptWrapperLine(lines[0], path))
{
lines.RemoveAt(0);
TrimOuterBlankLines(lines);
}
while (lines.Count > 0 && lines[0].Trim() == "---")
{
lines.RemoveAt(0);
TrimOuterBlankLines(lines);
}
while (lines.Count > 0 && IsCopiedPromptWrapperLine(lines[^1], path))
{
lines.RemoveAt(lines.Count - 1);
TrimOuterBlankLines(lines);
}
while (lines.Count > 0 && lines[^1].Trim() == "---")
{
lines.RemoveAt(lines.Count - 1);
TrimOuterBlankLines(lines);
}
return string.Join(Environment.NewLine, lines).TrimEnd();
}
static string StripWholeCodeFence(string content)
{
if (!content.StartsWith("```", StringComparison.Ordinal))
return content;
var firstNewline = content.IndexOf('\n');
var lastFence = content.LastIndexOf("```", StringComparison.Ordinal);
return firstNewline >= 0 && lastFence > firstNewline
? content[(firstNewline + 1)..lastFence].Trim()
: content;
}
static bool IsCopiedPromptWrapperLine(string line, string path)
{
var trimmed = line.Trim();
return trimmed.StartsWith("FILE:", StringComparison.OrdinalIgnoreCase)
|| trimmed.StartsWith("Memory file `", StringComparison.OrdinalIgnoreCase)
|| trimmed.StartsWith("[Memory file", StringComparison.OrdinalIgnoreCase)
|| trimmed.StartsWith("<memory-file", StringComparison.OrdinalIgnoreCase)
|| trimmed.StartsWith("</memory-file", StringComparison.OrdinalIgnoreCase)
|| trimmed.Equals(path, StringComparison.OrdinalIgnoreCase);
}
static void TrimOuterBlankLines(List<string> lines)
{
while (lines.Count > 0 && string.IsNullOrWhiteSpace(lines[0]))
lines.RemoveAt(0);
while (lines.Count > 0 && string.IsNullOrWhiteSpace(lines[^1]))
lines.RemoveAt(lines.Count - 1);
}
static string ResolveMemoryPath(string outputDir, string? relativePath)
{
var rel = NormalizeRelativePath(relativePath);
if (!rel.EndsWith(".md", StringComparison.OrdinalIgnoreCase))
throw new ArgumentException("Memory file path must end in .md");
if (rel.StartsWith("_trash/", StringComparison.OrdinalIgnoreCase))
throw new ArgumentException("Cannot write to _trash directly");
var full = Path.GetFullPath(Path.Combine(outputDir, rel));
var root = Path.GetFullPath(outputDir);
if (!full.StartsWith(root + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase))
throw new UnauthorizedAccessException("Memory path escapes output directory");
return full;
}
static string NormalizeRelativePath(string? relativePath)
{
if (string.IsNullOrWhiteSpace(relativePath))
throw new ArgumentException("Memory path is required");
var rel = relativePath.Replace('\\', '/').Trim().TrimStart('/');
var parts = rel.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (parts.Any(part => part == "." || part == ".."))
throw new ArgumentException("Memory path cannot contain . or .. segments");
return string.Join('/', parts);
}
static async Task<ChangeSetParseResult> ParseOrRepairChangeSetAsync(ModelClient client, string raw, string rawResponsePath)
{
try
{
return new ChangeSetParseResult(ParseChangeSet(raw), null);
}
catch (JsonException ex)
{
Console.WriteLine($" malformed JSON from model; requesting repair ({ex.Message})");
var repairPrompt = BuildJsonRepairPrompt(raw, ex.Message, MemoryChangeSetSchema());
var repairedRaw = await client.ChatAsync(repairPrompt);
var repairRawPath = Path.Combine(
Path.GetDirectoryName(rawResponsePath)!,
Path.GetFileNameWithoutExtension(rawResponsePath) + ".json-repair.raw.txt");
await File.WriteAllTextAsync(repairRawPath, repairedRaw);
try
{
return new ChangeSetParseResult(
ParseChangeSet(repairedRaw),
new JsonRepairCheckpoint(
OriginalError: ex.Message,
RawResponsePath: Path.GetFileName(rawResponsePath),
RepairRawResponsePath: Path.GetFileName(repairRawPath),
RepairRawResponse: repairedRaw,
Success: true,
Error: null));
}
catch (JsonException repairEx)
{
var failurePath = Path.Combine(
Path.GetDirectoryName(rawResponsePath)!,
Path.GetFileNameWithoutExtension(rawResponsePath) + ".json-repair.failed.txt");
await File.WriteAllTextAsync(failurePath, $"""
Original parse error:
{ex}
Repair parse error:
{repairEx}
Raw model response:
{raw}
Repair model response:
{repairedRaw}
""");
throw new InvalidOperationException(
$"Model returned invalid memory JSON and the repair attempt also failed. Raw responses were saved next to the checkpoint: {rawResponsePath}",
repairEx);
}
}
}
static async Task<SourceChangeAuditParseResult> ParseOrRepairSourceChangeAuditAsync(ModelClient client, string raw, string rawResponsePath)
{
try
{
return new SourceChangeAuditParseResult(ParseSourceChangeAuditReport(raw), null);
}
catch (JsonException ex)
{
Console.WriteLine($" malformed change-audit JSON; requesting repair ({ex.Message})");
var repairPrompt = BuildJsonRepairPrompt(raw, ex.Message, SourceChangeAuditSchema());
var repairedRaw = await client.ChatAsync(repairPrompt);
var repairRawPath = Path.Combine(
Path.GetDirectoryName(rawResponsePath)!,
Path.GetFileNameWithoutExtension(rawResponsePath) + ".json-repair.raw.txt");
await File.WriteAllTextAsync(repairRawPath, repairedRaw);
try
{
return new SourceChangeAuditParseResult(
ParseSourceChangeAuditReport(repairedRaw),
new JsonRepairCheckpoint(
OriginalError: ex.Message,
RawResponsePath: Path.GetFileName(rawResponsePath),
RepairRawResponsePath: Path.GetFileName(repairRawPath),
RepairRawResponse: repairedRaw,
Success: true,
Error: null));
}
catch (JsonException repairEx)
{
var failurePath = Path.Combine(
Path.GetDirectoryName(rawResponsePath)!,
Path.GetFileNameWithoutExtension(rawResponsePath) + ".json-repair.failed.txt");
await File.WriteAllTextAsync(failurePath, $"""
Original parse error:
{ex}
Repair parse error:
{repairEx}
Raw model response:
{raw}
Repair model response:
{repairedRaw}
""");
throw new InvalidOperationException(
$"Model returned invalid change-audit JSON and the repair attempt also failed. Raw responses were saved next to the checkpoint: {rawResponsePath}",
repairEx);
}
}
}
static async Task<HistoryUpdateParseResult> ParseOrRepairHistoryUpdateAsync(ModelClient client, string raw, string rawResponsePath)
{
try
{
return new HistoryUpdateParseResult(ParseHistoryUpdate(raw), null);
}
catch (JsonException ex)
{
Console.WriteLine($" malformed history JSON; requesting repair ({ex.Message})");
var repairPrompt = BuildJsonRepairPrompt(raw, ex.Message, HistoryUpdateSchema());
var repairedRaw = await client.ChatAsync(repairPrompt);
var repairRawPath = Path.Combine(
Path.GetDirectoryName(rawResponsePath)!,
Path.GetFileNameWithoutExtension(rawResponsePath) + ".json-repair.raw.txt");
await File.WriteAllTextAsync(repairRawPath, repairedRaw);
try
{
return new HistoryUpdateParseResult(
ParseHistoryUpdate(repairedRaw),
new JsonRepairCheckpoint(
OriginalError: ex.Message,
RawResponsePath: Path.GetFileName(rawResponsePath),
RepairRawResponsePath: Path.GetFileName(repairRawPath),
RepairRawResponse: repairedRaw,
Success: true,
Error: null));
}
catch (JsonException repairEx)
{
var failurePath = Path.Combine(
Path.GetDirectoryName(rawResponsePath)!,
Path.GetFileNameWithoutExtension(rawResponsePath) + ".json-repair.failed.txt");
await File.WriteAllTextAsync(failurePath, $"""
Original parse error:
{ex}
Repair parse error:
{repairEx}
Raw model response:
{raw}
Repair model response:
{repairedRaw}
""");
throw new InvalidOperationException(
$"Model returned invalid history JSON and the repair attempt also failed. Raw responses were saved next to the checkpoint: {rawResponsePath}",
repairEx);
}
}
}
static string BuildJsonRepairPrompt(string raw, string error, string schema)
{
return $$"""
Repair this malformed JSON response.
Requirements:
- Return ONLY valid JSON.
- Preserve all memory operation intent and markdown content.
- Do not summarize, delete, or invent content.
- Escape quotes, backslashes, and control characters correctly inside JSON strings.
- Remove any prose outside the JSON object.
- The output must match this schema:
{{schema}}
Parser error:
{{error}}
Malformed response begins:
{{raw}}
Malformed response ends.
""";
}
static string MemoryChangeSetSchema()
{
return """
{
"operations": [
{
"action": "save",
"path": "recent.md",
"content": "complete markdown content"
},
{
"action": "soft_delete",
"path": "obsolete.md",
"reason": "why this file is obsolete or superseded"
},
{
"action": "noop",
"reason": "why no memory change is needed"
}
]
}
""";
}
static string SourceChangeAuditSchema()
{
return """
{
"summary": "brief overview of what changed and whether anything appears lost",
"added": [
{
"fact": "new durable fact that appears in AFTER",
"file": "target file path",
"reason": "why this was added"
}
],
"changed": [
{
"before": "previous fact or wording",
"after": "new fact or wording",
"file": "file path where the change appears",
"reason": "why the change is justified"
}
],
"moved": [
{
"fact": "durable fact that moved or was merged",
"from": "previous file/path or broad location",
"to": "new file/path or broad location",
"reason": "why this is better categorized here"
}
],
"removed": [
{
"fact": "durable fact that appears absent from AFTER",
"from": "previous file/path or broad location",
"classification": "contradicted|obsolete|duplicate|not_durable|unjustified_loss_risk",
"justification": "specific reason this removal is acceptable, or why it is a risk",
"recommendedRepair": "what to restore if classification is unjustified_loss_risk"
}
],
"lossRisks": [
{
"fact": "missing or weakened durable fact",
"from": "previous file/path or broad location",
"risk": "why this may be accidental loss",
"recommendedRepair": "where/how to restore it"
}
],
"commitSummary": [
"short bullet suitable for a git commit body"
]
}
""";
}
static string HistoryUpdateSchema()
{
return """
{
"content": "complete markdown content for history.md",
"notes": [
"optional short note about important additions or corrections made in this update"
]
}
""";
}
static void CleanupParseTempFiles(string checkpointDir, string rawResponsePath, JsonRepairCheckpoint? repair)
{
TryDeleteFile(rawResponsePath);
if (repair is null)
return;
TryDeleteFile(Path.Combine(checkpointDir, repair.RepairRawResponsePath));
}
static void TryDeleteFile(string path)
{
if (string.IsNullOrWhiteSpace(path))
return;
try
{
if (File.Exists(path))
File.Delete(path);
}
catch
{
// Best-effort cleanup only. If deletion fails, keep the raw response for inspection.
}
}
static MemoryChangeSet ParseChangeSet(string raw)
{
var json = ExtractJson(raw);
return JsonSerializer.Deserialize<MemoryChangeSet>(json, Shared.JsonOptions)
?? new MemoryChangeSet { Operations = [new MemoryOperation { Action = "noop", Reason = "Empty model response" }] };
}
static SourceChangeAuditReport ParseSourceChangeAuditReport(string raw)
{
var json = ExtractJson(raw);
var report = JsonSerializer.Deserialize<SourceChangeAuditReport>(json, Shared.JsonOptions)
?? new SourceChangeAuditReport { Summary = "Empty change audit response." };
report.Added ??= new();
report.Changed ??= new();
report.Moved ??= new();
report.Removed ??= new();
report.LossRisks ??= new();
report.CommitSummary ??= new();
return report;
}
static HistoryUpdate ParseHistoryUpdate(string raw)
{
var json = ExtractJson(raw);
var update = JsonSerializer.Deserialize<HistoryUpdate>(json, Shared.JsonOptions)
?? new HistoryUpdate { Content = "" };
update.Notes ??= new();
return update;
}
static string ExtractJson(string raw)
{
raw = raw.Trim();
if (raw.StartsWith("```", StringComparison.Ordinal))
{
var firstNewline = raw.IndexOf('\n');
var lastFence = raw.LastIndexOf("```", StringComparison.Ordinal);
if (firstNewline >= 0 && lastFence > firstNewline)
raw = raw[(firstNewline + 1)..lastFence].Trim();
}
var firstBrace = raw.IndexOf('{');
var lastBrace = raw.LastIndexOf('}');
if (firstBrace >= 0 && lastBrace > firstBrace)
return raw[firstBrace..(lastBrace + 1)];
return raw;
}
static int EstimateTokens(string text) => (int)Math.Ceiling(text.Length * Shared.TokensPerCharacter);
static int LoadOrder(string relativePath)
{
var preferred = new[]
{
"core.md", "recent.md", "history.md", "history.rebuild.md", "communication.md", "raymond.md", "kai.md",
"health.md", "story-world.md", "projects.md", "preferences.md",
"index.md", "manifest.md"
};
var idx = Array.FindIndex(preferred, p => string.Equals(p, relativePath, StringComparison.OrdinalIgnoreCase));
return idx >= 0 ? idx : 1_000;
}
static class Shared
{
public static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
WriteIndented = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public const double TokensPerCharacter = 0.3;
}
sealed record SourceConversation(string Key, string Path, int Order, int? PartNumber, string DisplayName, string Slug)
{
public static SourceConversation FromAdHocPath(string path)
{
var id = System.IO.Path.GetFileNameWithoutExtension(path);
var display = $"KaiChat Conversation {id}";
var slug = Slugify(display);
return new($"current_{slug}", path, int.MaxValue, null, display, slug);
}
public static SourceConversation? FromPath(string path, int order)
{
var file = System.IO.Path.GetFileName(path);
if (!File.Exists(path))
return null;
var part = Regex.Match(file, @"^(\d{3})_The Chat Part (\d+)\.md$", RegexOptions.IgnoreCase);
if (part.Success && int.TryParse(part.Groups[2].Value, out var partNumber))
{
var display = $"The Chat Part {partNumber}";
var slug = Slugify(display);
return new($"{order:D3}_{slug}", path, order, partNumber, display, slug);
}
var test = Regex.Match(file, @"^T(\d{2})_Test Chat (\d+)\.md$", RegexOptions.IgnoreCase);
if (test.Success && int.TryParse(test.Groups[2].Value, out var testNumber))
{
var display = $"Test Chat {testNumber}";
var slug = Slugify(display);
return new($"{order:D3}_{slug}", path, order, null, display, slug);
}
return null;
}
private static string Slugify(string value)
{
value = Regex.Replace(value.ToLowerInvariant(), @"[^a-z0-9]+", "-").Trim('-');
return string.IsNullOrWhiteSpace(value) ? "conversation" : value;
}
}
sealed class Options
{
public bool Apply { get; init; }
public bool Resume { get; init; } = true;
public bool SkipSnapshotted { get; init; }
public bool CommitAfterSource { get; init; }
public bool RetentionAudit { get; init; } = true;
public bool ChangeAudit { get; init; } = true;
public bool HistoryPass { get; init; }
public bool HistoryFragmentPass { get; init; }
public bool HistoryMergePass { get; init; }
public bool SkipHistoryCheckpointed { get; init; }
public string BaseDir { get; init; } = Environment.CurrentDirectory;
public string? SourceFile { get; init; }
public string? IncludeCurrentSourceFile { get; init; }
public string HistorySeedPath { get; init; } = "history.md";
public string HistoryOutputPath { get; init; } = "history.rebuild.md";
public string SourceDir { get; init; } = "Extracted The Chat Project";
public string OutputDir { get; init; } = Path.Combine("KaiChat", "Memory_Rebuild");
public int ChunkTokens { get; init; } = 600_000;
public int MaxOutputTokens { get; init; } = 384_000;
public int RequestIdleTimeoutMinutes { get; init; } = 10;
public int MaxChunks { get; init; }
public int? FromPart { get; init; }
public int? ToPart { get; init; }
public string? SourceKey { get; init; }
public int MaxChunkChars => Math.Max(4_000, (int)(ChunkTokens / Shared.TokensPerCharacter));
public static Options Parse(string[] args)
{
var opts = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < args.Length; i++)
{
var arg = args[i];
if (!arg.StartsWith("--", StringComparison.Ordinal))
continue;
var key = arg[2..];
if (key is "apply" or "no-resume" or "skip-snapshotted" or "commit-after-source" or "no-retention-audit" or "no-change-audit" or "history-pass" or "history-fragment-pass" or "history-merge-pass" or "skip-history-checkpointed")
{
opts[key] = "true";
continue;
}
if (i + 1 >= args.Length)
throw new ArgumentException($"Missing value for {arg}");
opts[key] = args[++i];
}
return new Options
{
Apply = opts.ContainsKey("apply"),
Resume = !opts.ContainsKey("no-resume"),
SkipSnapshotted = opts.ContainsKey("skip-snapshotted"),
CommitAfterSource = opts.ContainsKey("commit-after-source"),
RetentionAudit = !opts.ContainsKey("no-retention-audit"),
ChangeAudit = !opts.ContainsKey("no-change-audit"),
HistoryPass = opts.ContainsKey("history-pass"),
HistoryFragmentPass = opts.ContainsKey("history-fragment-pass"),
HistoryMergePass = opts.ContainsKey("history-merge-pass"),
SkipHistoryCheckpointed = opts.ContainsKey("skip-history-checkpointed"),
BaseDir = opts.GetValueOrDefault("base") ?? Environment.CurrentDirectory,
SourceFile = opts.GetValueOrDefault("source-file"),
IncludeCurrentSourceFile = opts.GetValueOrDefault("include-current-source-file"),
HistorySeedPath = opts.GetValueOrDefault("history-seed") ?? "history.md",
HistoryOutputPath = opts.GetValueOrDefault("history-output") ?? "history.rebuild.md",
SourceDir = opts.GetValueOrDefault("source") ?? "Extracted The Chat Project",
OutputDir = opts.GetValueOrDefault("output") ?? Path.Combine("KaiChat", "Memory_Rebuild"),
ChunkTokens = ParseInt(opts, "chunk-tokens", 600_000),
MaxOutputTokens = ParseInt(opts, "max-output-tokens", 384_000),
RequestIdleTimeoutMinutes = ParseInt(opts, "request-idle-timeout-minutes", ParseInt(opts, "request-timeout-minutes", 10)),
MaxChunks = ParseInt(opts, "max-chunks", 0),
FromPart = ParseNullableInt(opts, "from"),
ToPart = ParseNullableInt(opts, "to"),
SourceKey = opts.GetValueOrDefault("source-key")
};
}
private static int ParseInt(Dictionary<string, string?> opts, string key, int fallback)
{
return opts.TryGetValue(key, out var value) && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)
? parsed
: fallback;
}
private static int? ParseNullableInt(Dictionary<string, string?> opts, string key)
{
return opts.TryGetValue(key, out var value) && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)
? parsed
: null;
}
}
sealed class ModelClient
{
private readonly HttpClient _http = new();
private readonly string _endpoint;
private readonly string _model;
private readonly int _maxOutputTokens;
private readonly int _requestIdleTimeoutMinutes;
private ModelClient(string endpoint, string apiKey, string model, int maxOutputTokens, int requestIdleTimeoutMinutes)
{
_endpoint = endpoint;
_model = model;
_maxOutputTokens = maxOutputTokens;
_requestIdleTimeoutMinutes = requestIdleTimeoutMinutes;
_http.Timeout = Timeout.InfiniteTimeSpan;
if (!string.IsNullOrWhiteSpace(apiKey))
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
}
public static async Task<ModelClient> CreateAsync(string baseDir, Options options)
{
var appSettingsPath = Path.Combine(baseDir, "KaiChat", "appsettings.json");
using var doc = JsonDocument.Parse(await File.ReadAllTextAsync(appSettingsPath));
var kai = doc.RootElement.GetProperty("KaiChat");
return new ModelClient(
kai.GetProperty("ApiEndpoint").GetString() ?? "",
kai.GetProperty("ApiKey").GetString() ?? "",
kai.GetProperty("Model").GetString() ?? "",
options.MaxOutputTokens,
options.RequestIdleTimeoutMinutes);
}
public async Task<string> ChatAsync(string prompt)
{
var requestBody = new
{
model = _model,
messages = new[]
{
new { role = "system", content = prompt }
},
stream = true,
max_tokens = _maxOutputTokens
};
var json = JsonSerializer.Serialize(requestBody, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
});
using var idleCts = new CancellationTokenSource();
ResetIdleTimeout(idleCts);
try
{
using var request = new HttpRequestMessage(HttpMethod.Post, _endpoint)
{
Content = new StringContent(json, Encoding.UTF8, "application/json")
};
using var response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, idleCts.Token);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(idleCts.Token);
using var reader = new StreamReader(stream);
var contentBuilder = new StringBuilder();
while (true)
{
var line = await reader.ReadLineAsync(idleCts.Token);
if (line is null)
break;
if (line.Length == 0 || !line.StartsWith("data: ", StringComparison.Ordinal))
continue;
var data = line[6..].Trim();
if (data == "[DONE]")
break;
var tokenReceived = false;
try
{
using var doc = JsonDocument.Parse(data);
if (!doc.RootElement.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0)
continue;
var choice = choices[0];
if (!choice.TryGetProperty("delta", out var delta))
continue;
if (delta.TryGetProperty("content", out var content))
{
var text = content.GetString();
if (!string.IsNullOrEmpty(text))
{
contentBuilder.Append(text);
tokenReceived = true;
}
}
if (delta.TryGetProperty("reasoning_content", out var reasoning)
&& !string.IsNullOrEmpty(reasoning.GetString()))
{
tokenReceived = true;
}
}
catch (JsonException)
{
// Ignore malformed stream fragments; the completed content is validated later.
}
if (tokenReceived)
ResetIdleTimeout(idleCts);
}
return contentBuilder.ToString();
}
catch (OperationCanceledException) when (_requestIdleTimeoutMinutes > 0 && idleCts.IsCancellationRequested)
{
throw new TimeoutException($"Model stream was idle for more than {_requestIdleTimeoutMinutes} minute(s).");
}
}
private void ResetIdleTimeout(CancellationTokenSource cts)
{
if (_requestIdleTimeoutMinutes > 0)
cts.CancelAfter(TimeSpan.FromMinutes(_requestIdleTimeoutMinutes));
}
}
sealed class MemoryChangeSet
{
[JsonPropertyName("operations")]
public List<MemoryOperation> Operations { get; set; } = new();
}
sealed class MemoryOperation
{
[JsonPropertyName("action")]
public string Action { get; set; } = "noop";
[JsonPropertyName("path")]
public string? Path { get; set; }
[JsonPropertyName("content")]
public string? Content { get; set; }
[JsonPropertyName("reason")]
public string? Reason { get; set; }
}
sealed record OperationResult(string Action, string Path, bool Success, string Message);
sealed record ChangeSetParseResult(
MemoryChangeSet ChangeSet,
JsonRepairCheckpoint? Repair);
sealed record SourceChangeAuditParseResult(
SourceChangeAuditReport Report,
JsonRepairCheckpoint? Repair);
sealed record HistoryUpdateParseResult(
HistoryUpdate Update,
JsonRepairCheckpoint? Repair);
sealed class HistoryUpdate
{
[JsonPropertyName("content")]
public string? Content { get; set; }
[JsonPropertyName("notes")]
public List<string>? Notes { get; set; }
}
sealed class SourceChangeAuditReport
{
[JsonPropertyName("source")]
public string Source { get; set; } = "";
[JsonPropertyName("sourceKey")]
public string SourceKey { get; set; } = "";
[JsonPropertyName("createdAtUtc")]
public DateTime CreatedAtUtc { get; set; }
[JsonPropertyName("summary")]
public string Summary { get; set; } = "";
[JsonPropertyName("added")]
public List<AddedChangeItem> Added { get; set; } = new();
[JsonPropertyName("changed")]
public List<ChangedChangeItem> Changed { get; set; } = new();
[JsonPropertyName("moved")]
public List<MovedChangeItem> Moved { get; set; } = new();
[JsonPropertyName("removed")]
public List<RemovedChangeItem> Removed { get; set; } = new();
[JsonPropertyName("lossRisks")]
public List<LossRiskItem> LossRisks { get; set; } = new();
[JsonPropertyName("commitSummary")]
public List<string> CommitSummary { get; set; } = new();
[JsonPropertyName("jsonRepair")]
public JsonRepairCheckpoint? JsonRepair { get; set; }
}
sealed class AddedChangeItem
{
[JsonPropertyName("fact")]
public string Fact { get; set; } = "";
[JsonPropertyName("file")]
public string File { get; set; } = "";
[JsonPropertyName("reason")]
public string Reason { get; set; } = "";
}
sealed class ChangedChangeItem
{
[JsonPropertyName("before")]
public string Before { get; set; } = "";
[JsonPropertyName("after")]
public string After { get; set; } = "";
[JsonPropertyName("file")]
public string File { get; set; } = "";
[JsonPropertyName("reason")]
public string Reason { get; set; } = "";
}
sealed class MovedChangeItem
{
[JsonPropertyName("fact")]
public string Fact { get; set; } = "";
[JsonPropertyName("from")]
public string From { get; set; } = "";
[JsonPropertyName("to")]
public string To { get; set; } = "";
[JsonPropertyName("reason")]
public string Reason { get; set; } = "";
}
sealed class RemovedChangeItem
{
[JsonPropertyName("fact")]
public string Fact { get; set; } = "";
[JsonPropertyName("from")]
public string From { get; set; } = "";
[JsonPropertyName("classification")]
public string Classification { get; set; } = "";
[JsonPropertyName("justification")]
public string Justification { get; set; } = "";
[JsonPropertyName("recommendedRepair")]
public string RecommendedRepair { get; set; } = "";
}
sealed class LossRiskItem
{
[JsonPropertyName("fact")]
public string Fact { get; set; } = "";
[JsonPropertyName("from")]
public string From { get; set; } = "";
[JsonPropertyName("risk")]
public string Risk { get; set; } = "";
[JsonPropertyName("recommendedRepair")]
public string RecommendedRepair { get; set; } = "";
}
sealed record JsonRepairCheckpoint(
string OriginalError,
string RawResponsePath,
string RepairRawResponsePath,
string RepairRawResponse,
bool Success,
string? Error);
sealed record RetentionAuditCheckpoint(
string RawResponsePath,
string RawResponse,
JsonRepairCheckpoint? JsonRepair,
IReadOnlyList<OperationResult> Operations);
sealed record ProcessResult(int ExitCode, string Stdout, string Stderr)
{
public string CombinedOutput => string.Join(
Environment.NewLine,
new[] { Stdout.Trim(), Stderr.Trim() }.Where(s => !string.IsNullOrWhiteSpace(s)));
}
sealed record ChunkCheckpoint(
string Source,
string SourcePath,
int ChunkIndex,
int ChunkCount,
int EstimatedTokens,
DateTime CreatedAtUtc,
string RawResponse,
JsonRepairCheckpoint? JsonRepair,
IReadOnlyList<OperationResult> Operations,
RetentionAuditCheckpoint? RetentionAudit);
sealed record HistoryCheckpoint(
string Source,
string SourcePath,
int ChunkIndex,
int ChunkCount,
int EstimatedTokens,
DateTime CreatedAtUtc,
string RawResponse,
JsonRepairCheckpoint? JsonRepair,
IReadOnlyList<string> Notes,
int HistoryCharacters);
sealed record HistoryFragment(string Name, string Content);
sealed record HistoryFragmentCheckpoint(
string Source,
string SourcePath,
int ChunkIndex,
int ChunkCount,
int EstimatedTokens,
DateTime CreatedAtUtc,
string RawResponse,
string FragmentPath,
int FragmentCharacters);