CompactionService.cs

using System.Security.Cryptography;
using System.Text;
using KaiChat.Models;

namespace KaiChat.Services;

public interface ICompactionService
{
    Task<CompactionResult> CompactIfNeededAsync(
        Conversation conv,
        string systemPrompt,
        bool force = false,
        CompactionProgressHandler? progress = null,
        string? conversationsFolder = null,
        CancellationToken ct = default);

    Task<CompactionPreview> PreviewCompactionAsync(
        Conversation conv,
        string systemPrompt,
        bool force = false,
        string? conversationsFolder = null);

    Task<CompactionResult> TestCompactionAsync(
        Conversation conv,
        string systemPrompt,
        bool force = false,
        bool generateSummary = false,
        CompactionProgressHandler? progress = null,
        string? conversationsFolder = null,
        CancellationToken ct = default);

    Task<List<ChatMessage>> BuildApiMessagesAsync(Conversation conv, string systemPrompt);
    int EstimateTokenCount(string text);
    int EstimateMessagesTokenCount(IEnumerable<ChatMessage> messages);
    int GetTriggerTokenCount();
}

public delegate Task CompactionProgressHandler(CompactionProgressEvent evt);

public class CompactionProgressEvent
{
    public string Type { get; set; } = "";
    public string Message { get; set; } = "";
    public string Text { get; set; } = "";
    public int? ChunkIndex { get; set; }
    public int? ChunkCount { get; set; }
}

public class CompactionPreview
{
    public bool Enabled { get; set; }
    public bool Force { get; set; }
    public bool ShouldCompact { get; set; }
    public string Reason { get; set; } = "";
    public int ModelInputTokens { get; set; }
    public int ModelOutputTokens { get; set; }
    public int ContextTokenLimit { get; set; }
    public int TriggerTokens { get; set; }
    public int RecentContextTargetTokens { get; set; }
    public int ChunkTokenLimit { get; set; }
    public int MergeBatchTokenLimit { get; set; }
    public int SummaryMaxTokens { get; set; }
    public int EstimatedApiTokens { get; set; }
    public int EstimatedPostCompactionApiTokens { get; set; }
    public int EstimatedMessagesTokens { get; set; }
    public int EstimatedCompactedTokens { get; set; }
    public int EstimatedKeptTokens { get; set; }
    public int EstimatedSummaryTokens { get; set; }
    public int MessagesBefore { get; set; }
    public int MessagesToCompact { get; set; }
    public int MessagesToKeep { get; set; }
    public int ChunkCount { get; set; }
    public List<CompactionChunkInfo> Chunks { get; set; } = new();
}

public class CompactionResult : CompactionPreview
{
    public bool Compacted { get; set; }
    public bool DryRun { get; set; }
    public bool GeneratedSummary { get; set; }
    public int CompactionNumber { get; set; }
    public string? ArchivePath { get; set; }
    public string? PreviousArchivePath { get; set; }
    public string? SummaryPreview { get; set; }
    public string? Summary { get; set; }
    public string? Thinking { get; set; }
    public List<string> ArchiveChain { get; set; } = new();
}

public class CompactionChunkInfo
{
    public int Index { get; set; }
    public int UnitCount { get; set; }
    public int EstimatedTokens { get; set; }
    public string? FirstMessageId { get; set; }
    public string? LastMessageId { get; set; }
}

public class CompactionService : ICompactionService
{
    private readonly IChatService _chat;
    private readonly IConversationService _convService;
    private readonly ISearchService _search;
    private readonly KaiChatConfig _config;
    private readonly string _baseFolder;
    private readonly string _archiveDir;
    private readonly ILogger<CompactionService> _logger;

    public CompactionService(
        IChatService chat,
        IConversationService convService,
        ISearchService search,
        IConfiguration config,
        KaiChatConfig kaiChatConfig,
        ILogger<CompactionService> logger)
    {
        _chat = chat;
        _convService = convService;
        _search = search;
        _config = kaiChatConfig;
        _baseFolder = config.GetValue<string>("KaiChat:BaseFolder") ?? ".";
        _archiveDir = Path.Combine(_baseFolder, "KaiChat", "Archive");
        Directory.CreateDirectory(_archiveDir);
        _logger = logger;
    }

    public int EstimateTokenCount(string text)
    {
        if (string.IsNullOrEmpty(text)) return 0;
        return (int)Math.Ceiling(text.Length / 3.5);
    }

    public int EstimateMessagesTokenCount(IEnumerable<ChatMessage> messages)
    {
        var total = 0;
        foreach (var msg in messages)
            total += EstimateTokenCount(msg.Role) + EstimateTokenCount(msg.Content) + 6;
        return total;
    }

    public int GetTriggerTokenCount()
    {
        var settings = Settings();
        if (settings.TriggerTokens > 0)
            return settings.TriggerTokens;

        var limit = GetContextTokenLimit();
        var percent = Math.Clamp(settings.TriggerPercent, 1, 100);
        return Math.Max(1, (int)Math.Floor(limit * (percent / 100.0)));
    }

    public Task<CompactionPreview> PreviewCompactionAsync(
        Conversation conv,
        string systemPrompt,
        bool force = false,
        string? conversationsFolder = null)
    {
        var plan = BuildPlan(conv, systemPrompt, force);
        return Task.FromResult(plan.Preview);
    }

    public async Task<CompactionResult> TestCompactionAsync(
        Conversation conv,
        string systemPrompt,
        bool force = false,
        bool generateSummary = false,
        CompactionProgressHandler? progress = null,
        string? conversationsFolder = null,
        CancellationToken ct = default)
    {
        return await RunCompactionAsync(conv, systemPrompt, force, dryRun: true, generateSummary, progress, conversationsFolder, ct);
    }

    public async Task<CompactionResult> CompactIfNeededAsync(
        Conversation conv,
        string systemPrompt,
        bool force = false,
        CompactionProgressHandler? progress = null,
        string? conversationsFolder = null,
        CancellationToken ct = default)
    {
        return await RunCompactionAsync(conv, systemPrompt, force, dryRun: false, generateSummary: true, progress, conversationsFolder, ct);
    }

    public Task<List<ChatMessage>> BuildApiMessagesAsync(Conversation conv, string systemPrompt)
    {
        return Task.FromResult(BuildApiMessages(
            systemPrompt,
            conv.CompactedSummary,
            conv.CompactionCount,
            conv.Messages));
    }

    private async Task<CompactionResult> RunCompactionAsync(
        Conversation conv,
        string systemPrompt,
        bool force,
        bool dryRun,
        bool generateSummary,
        CompactionProgressHandler? progress,
        string? conversationsFolder,
        CancellationToken ct)
    {
        var plan = BuildPlan(conv, systemPrompt, force);
        var result = ToResult(plan.Preview, dryRun);
        result.CompactionNumber = conv.CompactionCount + 1;
        result.PreviousArchivePath = conv.CompactionArchives.LastOrDefault()?.ArchivePath;
        result.ArchiveChain = conv.CompactionArchives.Select(a => a.ArchivePath).ToList();

        if (!plan.Preview.ShouldCompact)
            return result;

        _logger.LogInformation(
            "Compaction planned for {Id} ({Title}): api ~{ApiTokens:N0}, compacting {Count} messages in {Chunks} chunks",
            conv.Id,
            conv.Title,
            plan.Preview.EstimatedApiTokens,
            plan.Preview.MessagesToCompact,
            plan.Preview.ChunkCount);

        if (!generateSummary)
            return result;

        var transcript = await BuildCompactedSummaryAsync(conv, plan, progress, ct);
        var summary = transcript.Summary;
        result.GeneratedSummary = true;
        result.Summary = summary;
        result.Thinking = transcript.Thinking;
        result.SummaryPreview = summary.Length > 2000 ? summary[..2000] + "..." : summary;

        if (dryRun)
            return result;

        var archiveEntry = await ArchiveAsync(conv, plan.ToCompact, summary, plan.Preview.EstimatedCompactedTokens, conversationsFolder, ct);

        conv.CompactedSummary = summary;
        conv.CompactionCount++;
        conv.Messages = plan.Keep;
        conv.CompactionArchives.Add(archiveEntry);

        await _convService.SaveConversationAsync(conv, conversationsFolder);
        await _search.RebuildIndexAsync();

        result.Compacted = true;
        result.ArchivePath = archiveEntry.ArchivePath;
        result.PreviousArchivePath = archiveEntry.PreviousArchivePath;
        result.ArchiveChain.Add(archiveEntry.ArchivePath);

        _logger.LogInformation(
            "Compaction #{Count} complete for {Id}: kept {Keep} messages, archive {Archive}",
            conv.CompactionCount,
            conv.Id,
            plan.Keep.Count,
            archiveEntry.ArchivePath);

        return result;
    }

    private CompactionPlan BuildPlan(Conversation conv, string systemPrompt, bool force)
    {
        var settings = Settings();
        var contextTokenLimit = GetContextTokenLimit();
        var chunkTokenLimit = GetChunkTokenLimit();
        var recentContextTargetTokens = GetRecentContextTargetTokenCount();
        var mergeBatchTokenLimit = GetMergeBatchTokenLimit();
        var summaryMaxTokens = GetSummaryMaxTokens();
        var apiMessages = BuildApiMessages(systemPrompt, conv.CompactedSummary, conv.CompactionCount, conv.Messages);
        var estimatedApiTokens = EstimateMessagesTokenCount(apiMessages);
        var estimatedMessageTokens = EstimateMessagesTokenCount(conv.Messages);
        var triggerTokens = GetTriggerTokenCount();
        var keep = SelectMessagesToKeep(conv.Messages, force);
        var compactCount = Math.Max(0, conv.Messages.Count - keep.Count);
        var toCompact = compactCount > 0
            ? conv.Messages.Take(compactCount).ToList()
            : new List<ChatMessage>();

        var units = BuildCompactionUnits(toCompact);
        var chunks = BuildChunks(units, chunkTokenLimit);
        var compactedTokens = units.Sum(u => u.EstimatedTokens);
        var keptTokens = EstimateMessagesTokenCount(keep);
        var estimatedSummaryTokens = toCompact.Count == 0
            ? EstimateTokenCount(conv.CompactedSummary ?? "")
            : Math.Min(
                summaryMaxTokens,
                Math.Max(1_000, compactedTokens / 8));
        var syntheticSummary = new string('x', Math.Max(1, estimatedSummaryTokens * 4));
        var postMessages = BuildApiMessages(
            systemPrompt,
            syntheticSummary,
            conv.CompactionCount + (toCompact.Count > 0 ? 1 : 0),
            keep);

        var shouldCompact = force || (settings.Enabled && estimatedApiTokens >= triggerTokens);
        var reason = shouldCompact
            ? force
                ? "Forced compaction requested."
                : $"Estimated API payload is at or above the compaction trigger ({estimatedApiTokens:N0}/{triggerTokens:N0})."
            : $"Estimated API payload is below the compaction trigger ({estimatedApiTokens:N0}/{triggerTokens:N0}).";

        if (!settings.Enabled && !force)
        {
            shouldCompact = false;
            reason = "Compaction is disabled in configuration.";
        }

        if (shouldCompact && toCompact.Count == 0)
        {
            shouldCompact = false;
            reason = "No older messages can be compacted while preserving the recent context target.";
        }

        return new CompactionPlan
        {
            Preview = new CompactionPreview
            {
                Enabled = settings.Enabled,
                Force = force,
                ShouldCompact = shouldCompact,
                Reason = reason,
                ModelInputTokens = _config.GetModelInputTokenLimit(),
                ModelOutputTokens = _config.GetModelOutputTokenLimit(),
                ContextTokenLimit = contextTokenLimit,
                TriggerTokens = triggerTokens,
                RecentContextTargetTokens = recentContextTargetTokens,
                ChunkTokenLimit = chunkTokenLimit,
                MergeBatchTokenLimit = mergeBatchTokenLimit,
                SummaryMaxTokens = summaryMaxTokens,
                EstimatedApiTokens = estimatedApiTokens,
                EstimatedPostCompactionApiTokens = EstimateMessagesTokenCount(postMessages),
                EstimatedMessagesTokens = estimatedMessageTokens,
                EstimatedCompactedTokens = compactedTokens,
                EstimatedKeptTokens = keptTokens,
                EstimatedSummaryTokens = estimatedSummaryTokens,
                MessagesBefore = conv.Messages.Count,
                MessagesToCompact = toCompact.Count,
                MessagesToKeep = keep.Count,
                ChunkCount = chunks.Count,
                Chunks = chunks.Select((chunk, index) => new CompactionChunkInfo
                {
                    Index = index + 1,
                    UnitCount = chunk.Count,
                    EstimatedTokens = chunk.Sum(u => u.EstimatedTokens),
                    FirstMessageId = chunk.FirstOrDefault()?.MessageId,
                    LastMessageId = chunk.LastOrDefault()?.MessageId
                }).ToList()
            },
            ToCompact = toCompact,
            Keep = keep,
            Chunks = chunks
        };
    }

    private async Task<CompactionTranscript> BuildCompactedSummaryAsync(
        Conversation conv,
        CompactionPlan plan,
        CompactionProgressHandler? progress,
        CancellationToken ct)
    {
        var summaryMaxTokens = GetSummaryMaxTokens();
        var chunkSummaries = new List<string>();
        var thinking = new StringBuilder();

        for (var i = 0; i < plan.Chunks.Count; i++)
        {
            ct.ThrowIfCancellationRequested();
            await ReportAsync(progress, new()
            {
                Type = "chunk-started",
                Message = $"Summarizing chunk {i + 1} of {plan.Chunks.Count}",
                ChunkIndex = i + 1,
                ChunkCount = plan.Chunks.Count
            });

            var chunkText = string.Join("\n\n", plan.Chunks[i].Select(u => u.Text));
            var messages = new List<ChatMessage>
            {
                new()
                {
                    Role = "system",
                    Content = "You are a KaiChat conversation compaction agent. Summarize this chronological slice for later continuation. Preserve exact facts, user corrections, decisions, relationship/lore context, preferences, unresolved tasks, timestamps, message ids when important, and any tool results. Do not invent details. Do not omit corrections or safety-critical context."
                },
                new()
                {
                    Role = "user",
                    Content = $"""
Conversation id: {conv.Id}
Conversation title: {conv.Title}
Compaction number: {conv.CompactionCount + 1}
Chunk: {i + 1} of {plan.Chunks.Count}

Messages in this chunk:

{chunkText}
"""
                }
            };

            await ReportAsync(progress, new()
            {
                Type = "summary-label",
                Text = $"\n\n## Chunk {i + 1} of {plan.Chunks.Count}\n\n",
                ChunkIndex = i + 1,
                ChunkCount = plan.Chunks.Count
            });

            var transcript = await StreamCompactionCallAsync(messages, summaryMaxTokens, progress, ct);
            thinking.Append(transcript.Thinking);
            var summary = transcript.Summary;
            if (string.IsNullOrWhiteSpace(summary))
                throw new InvalidOperationException($"Compaction chunk {i + 1} returned an empty summary.");

            chunkSummaries.Add(summary.Trim());
            await ReportAsync(progress, new()
            {
                Type = "chunk-complete",
                Message = $"Chunk {i + 1} summarized",
                ChunkIndex = i + 1,
                ChunkCount = plan.Chunks.Count
            });
        }

        if (string.IsNullOrWhiteSpace(conv.CompactedSummary) && chunkSummaries.Count == 1)
            return new CompactionTranscript(chunkSummaries[0], thinking.ToString());

        var merged = await MergeSummariesAsync(conv, chunkSummaries, thinking, progress, ct);
        return merged;
    }

    private async Task<CompactionTranscript> MergeSummariesAsync(
        Conversation conv,
        List<string> newSummaries,
        StringBuilder thinking,
        CompactionProgressHandler? progress,
        CancellationToken ct)
    {
        var mergeBatchTokenLimit = GetMergeBatchTokenLimit();
        var summaryMaxTokens = GetSummaryMaxTokens();
        var rollingSummary = conv.CompactedSummary ?? "";
        var pending = new Queue<string>(newSummaries);
        var mergeIndex = 0;

        while (pending.Count > 0)
        {
            mergeIndex++;
            var batch = new List<string>();
            var runningTokens = EstimateTokenCount(rollingSummary);
            while (pending.Count > 0)
            {
                var next = pending.Peek();
                var nextTokens = EstimateTokenCount(next);
                if (batch.Count > 0 && runningTokens + nextTokens > mergeBatchTokenLimit)
                    break;

                batch.Add(pending.Dequeue());
                runningTokens += nextTokens;
            }

            var batchText = string.Join("\n\n---\n\n", batch.Select((s, i) => $"New chunk summary {i + 1}:\n{s}"));
            var messages = new List<ChatMessage>
            {
                new()
                {
                    Role = "system",
                    Content = "Merge KaiChat compaction summaries into one current compacted history. Preserve chronology, exact facts, user corrections, durable preferences, relationship/lore state, unresolved tasks, and important tool/file changes. Remove duplication, but do not drop unique details. Return only the merged summary."
                },
                new()
                {
                    Role = "user",
                    Content = $"""
Conversation id: {conv.Id}
Conversation title: {conv.Title}
Compaction number: {conv.CompactionCount + 1}

Previous compacted summary:
{(string.IsNullOrWhiteSpace(rollingSummary) ? "(none)" : rollingSummary)}

New summaries to merge:
{batchText}
"""
                }
            };

            await ReportAsync(progress, new()
            {
                Type = "merge-started",
                Message = $"Merging compaction summaries ({mergeIndex})"
            });
            await ReportAsync(progress, new()
            {
                Type = "summary-label",
                Text = $"\n\n## Merged Summary {mergeIndex}\n\n"
            });

            var transcript = await StreamCompactionCallAsync(messages, summaryMaxTokens, progress, ct);
            thinking.Append(transcript.Thinking);
            var merged = transcript.Summary;
            if (string.IsNullOrWhiteSpace(merged))
                throw new InvalidOperationException("Compaction summary merge returned an empty summary.");

            rollingSummary = merged.Trim();
        }

        return new CompactionTranscript(rollingSummary, thinking.ToString());
    }

    private async Task<CompactionTranscript> StreamCompactionCallAsync(
        List<ChatMessage> messages,
        int maxTokens,
        CompactionProgressHandler? progress,
        CancellationToken ct)
    {
        var summary = new StringBuilder();
        var thinking = new StringBuilder();

        await foreach (var token in _chat.StreamChatAsync(messages, ct, maxTokens))
        {
            if (token.StartsWith("R|"))
            {
                var thought = token[2..];
                if (string.IsNullOrEmpty(thought)) continue;
                thinking.Append(thought);
                await ReportAsync(progress, new() { Type = "thinking-delta", Text = thought });
            }
            else if (token.StartsWith("C|"))
            {
                var content = token[2..];
                if (string.IsNullOrEmpty(content)) continue;
                summary.Append(content);
                await ReportAsync(progress, new() { Type = "summary-delta", Text = content });
            }
        }

        return new CompactionTranscript(summary.ToString().Trim(), thinking.ToString());
    }

    private static async Task ReportAsync(
        CompactionProgressHandler? progress,
        CompactionProgressEvent evt)
    {
        if (progress != null)
            await progress(evt);
    }

    private async Task<CompactionArchiveEntry> ArchiveAsync(
        Conversation conv,
        List<ChatMessage> messages,
        string summary,
        int estimatedArchivedTokens,
        string? conversationsFolder,
        CancellationToken ct)
    {
        var now = DateTime.UtcNow;
        var compactionNumber = conv.CompactionCount + 1;
        var previousArchivePath = conv.CompactionArchives.LastOrDefault()?.ArchivePath;
        var summaryHash = HashSummary(summary);
        var title = SanitizeFilename(conv.Title, Settings().MaxArchiveTitleLength);
        var timestamp = now.ToString("yyyyMMdd-HHmmss");
        var filePath = Path.Combine(_archiveDir, $"{conv.Id}_c{compactionNumber:000}_{timestamp}_{title}.md");
        var tempPath = filePath + ".tmp";
        var relativePath = ToRelativePath(filePath);

        try
        {
            await using var writer = new StreamWriter(tempPath, append: false, Encoding.UTF8);
            await writer.WriteLineAsync("---");
            await writer.WriteLineAsync("type: kaichat-compaction-archive");
            await writer.WriteLineAsync($"conversation_id: {EscapeFrontMatter(conv.Id)}");
            await writer.WriteLineAsync($"conversation_title: {EscapeFrontMatter(conv.Title)}");
            await writer.WriteLineAsync($"compaction_number: {compactionNumber}");
            await writer.WriteLineAsync($"archive_path: {EscapeFrontMatter(relativePath)}");
            await writer.WriteLineAsync($"previous_archive_path: {EscapeFrontMatter(previousArchivePath ?? "")}");
            await writer.WriteLineAsync($"created_at_utc: {now:O}");
            await writer.WriteLineAsync($"first_archived_message_id: {EscapeFrontMatter(messages.FirstOrDefault()?.Id ?? "")}");
            await writer.WriteLineAsync($"last_archived_message_id: {EscapeFrontMatter(messages.LastOrDefault()?.Id ?? "")}");
            await writer.WriteLineAsync($"archived_message_count: {messages.Count}");
            await writer.WriteLineAsync($"estimated_archived_tokens: {estimatedArchivedTokens}");
            await writer.WriteLineAsync($"summary_sha256_16: {summaryHash}");
            await writer.WriteLineAsync("---");
            await writer.WriteLineAsync();
            await writer.WriteLineAsync($"# Compaction #{compactionNumber}: {conv.Title}");
            await writer.WriteLineAsync();
            await writer.WriteLineAsync("## Trace Chain");
            await writer.WriteLineAsync();
            await writer.WriteLineAsync($"- Conversation JSON: `{_convService.GetConversationRelativePath(conv.Id, conversationsFolder)}`");
            foreach (var entry in conv.CompactionArchives.OrderBy(a => a.CompactionNumber))
                await writer.WriteLineAsync($"- Previous compaction #{entry.CompactionNumber}: `{entry.ArchivePath}`");
            await writer.WriteLineAsync($"- This compaction #{compactionNumber}: `{relativePath}`");
            await writer.WriteLineAsync();

            if (!string.IsNullOrWhiteSpace(conv.CompactedSummary))
            {
                await writer.WriteLineAsync("## Previous Compacted Summary");
                await writer.WriteLineAsync();
                await writer.WriteLineAsync(conv.CompactedSummary);
                await writer.WriteLineAsync();
            }

            await writer.WriteLineAsync("## New Compacted Summary");
            await writer.WriteLineAsync();
            await writer.WriteLineAsync(summary);
            await writer.WriteLineAsync();
            await writer.WriteLineAsync("## Archived Messages");
            await writer.WriteLineAsync();

            foreach (var msg in messages)
            {
                ct.ThrowIfCancellationRequested();
                await writer.WriteLineAsync($"### Message {msg.Id}");
                await writer.WriteLineAsync();
                await writer.WriteLineAsync($"- Role: `{msg.Role}`");
                await writer.WriteLineAsync($"- Timestamp UTC: `{msg.Timestamp.ToUniversalTime():O}`");
                await writer.WriteLineAsync($"- Timestamp Local: `{msg.Timestamp.ToLocalTime():dddd dd/MM/yyyy hh:mm:ss tt}`");
                await writer.WriteLineAsync();
                await writer.WriteLineAsync("#### Content");
                await writer.WriteLineAsync();
                await writer.WriteLineAsync(msg.Content);
                await writer.WriteLineAsync();

                if (!string.IsNullOrWhiteSpace(msg.Thinking))
                {
                    await writer.WriteLineAsync("#### Thinking");
                    await writer.WriteLineAsync();
                    await writer.WriteLineAsync(msg.Thinking);
                    await writer.WriteLineAsync();
                }
            }
        }
        catch
        {
            if (File.Exists(tempPath))
                File.Delete(tempPath);
            throw;
        }

        if (File.Exists(filePath))
            throw new IOException($"Archive already exists: {relativePath}");

        File.Move(tempPath, filePath);

        return new CompactionArchiveEntry
        {
            CompactionNumber = compactionNumber,
            ArchivePath = relativePath,
            PreviousArchivePath = previousArchivePath,
            CreatedAt = now,
            ArchivedMessageCount = messages.Count,
            FirstArchivedMessageId = messages.FirstOrDefault()?.Id,
            LastArchivedMessageId = messages.LastOrDefault()?.Id,
            FirstArchivedTimestamp = messages.FirstOrDefault()?.Timestamp,
            LastArchivedTimestamp = messages.LastOrDefault()?.Timestamp,
            EstimatedArchivedTokens = estimatedArchivedTokens,
            SummaryHash = summaryHash
        };
    }

    private List<ChatMessage> SelectMessagesToKeep(List<ChatMessage> messages, bool force)
    {
        var settings = Settings();
        var minRecent = Math.Clamp(settings.MinRecentMessages, 1, Math.Max(1, messages.Count));
        if (force)
            return messages.TakeLast(minRecent).ToList();

        var targetTokens = GetRecentContextTargetTokenCount();
        var keep = new List<ChatMessage>();
        var runningTokens = 0;

        for (var i = messages.Count - 1; i >= 0; i--)
        {
            var tokens = EstimateMessageWithTimestampTokens(messages[i]);
            var mustKeep = keep.Count < minRecent;
            if (!mustKeep && runningTokens + tokens > targetTokens)
                break;

            keep.Insert(0, messages[i]);
            runningTokens += tokens;
        }

        return keep;
    }

    private List<CompactionUnit> BuildCompactionUnits(List<ChatMessage> messages)
    {
        var maxChars = Math.Max(4_000, GetChunkTokenLimit() * 3);
        var units = new List<CompactionUnit>();

        for (var i = 0; i < messages.Count; i++)
        {
            var formatted = FormatMessageForCompaction(messages[i], i + 1);
            if (formatted.Length <= maxChars)
            {
                units.Add(new CompactionUnit(messages[i].Id, formatted, EstimateTokenCount(formatted)));
                continue;
            }

            var parts = SplitByLength(formatted, maxChars);
            for (var p = 0; p < parts.Count; p++)
            {
                var text = $"Message {messages[i].Id} oversized archive part {p + 1} of {parts.Count}\n\n{parts[p]}";
                units.Add(new CompactionUnit(messages[i].Id, text, EstimateTokenCount(text)));
            }
        }

        return units;
    }

    private List<List<CompactionUnit>> BuildChunks(List<CompactionUnit> units, int maxTokens)
    {
        var chunks = new List<List<CompactionUnit>>();
        var current = new List<CompactionUnit>();
        var runningTokens = 0;

        foreach (var unit in units)
        {
            if (current.Count > 0 && runningTokens + unit.EstimatedTokens > maxTokens)
            {
                chunks.Add(current);
                current = new List<CompactionUnit>();
                runningTokens = 0;
            }

            current.Add(unit);
            runningTokens += unit.EstimatedTokens;
        }

        if (current.Count > 0)
            chunks.Add(current);

        return chunks;
    }

    private List<ChatMessage> BuildApiMessages(
        string systemPrompt,
        string? compactedSummary,
        int compactionCount,
        IEnumerable<ChatMessage> conversationMessages)
    {
        var messages = new List<ChatMessage>
        {
            new() { Role = "system", Content = systemPrompt }
        };

        if (!string.IsNullOrWhiteSpace(compactedSummary))
        {
            messages.Add(new()
            {
                Role = "assistant",
                Content = $"[Compacted history of previous conversation (compaction #{compactionCount})]\n\n{compactedSummary}"
            });

            messages.Add(new()
            {
                Role = "system",
                Content = "The message above is a compacted summary of earlier parts of this conversation. Use it for context. The messages below are the recent, uncompacted part of the conversation."
            });
        }

        messages.AddRange(conversationMessages.Where(IsApiMessageRole).Select(msg =>
        {
            var ts = msg.Timestamp.ToLocalTime().ToString("dddd dd/MM/yyyy hh:mm:ss tt");
            var hasNativeToolCallId = msg.Role == "tool" && !string.IsNullOrWhiteSpace(msg.Tool?.ToolCallId);
            var role = msg.Role == "tool" && !hasNativeToolCallId ? "system" : msg.Role;
            var timedContent = msg.Role switch
            {
                "user" => $"[User message written at {ts}]\n{msg.Content}",
                "tool" => FormatToolMessageForApi(msg),
                "assistant" => ToolResultTextSanitizer.RemoveEchoedToolResultBlocks(msg.Content),
                _ => msg.Content
            };
            return new ChatMessage
            {
                Id = msg.Id,
                Role = role,
                Content = timedContent,
                Thinking = msg.Thinking,
                Tool = msg.Tool,
                ToolCalls = msg.ToolCalls.Select(call => new StoredToolCall
                {
                    Id = call.Id,
                    Name = call.Name,
                    Arguments = call.Arguments
                }).ToList(),
                Timestamp = msg.Timestamp
            };
        }));

        return messages;
    }

    private static bool IsApiMessageRole(ChatMessage msg)
    {
        return msg.Role is "user" or "assistant" or "system" or "tool";
    }

    private static string FormatToolMessageForApi(ChatMessage msg)
    {
        if (msg.Tool == null)
            return msg.Content;

        return $"""
        PRIVATE TOOL OUTPUT - DO NOT REPEAT IN CHAT (round {msg.Tool.Round})
        Instruction: Use this only as hidden context. Never copy this wrapper, the parameters, or the full output into a visible assistant reply.
        Reuse: If you need this same method/path/parameters again in this response chain, use this output instead of repeating the tool call unless Raymond asked for a refresh, a write changed the source, the previous result failed, or genuinely new data is needed.
        Grounding: {BuildToolGroundingInstruction(msg.Tool)}

        Tool: {msg.Tool.Method} {msg.Tool.Path}
        Status: {msg.Tool.Status}
        Parameters JSON:
        {msg.Tool.Body}
        Output:
        {msg.Tool.Result}
        END PRIVATE TOOL OUTPUT
        """;
    }

    private static string BuildToolGroundingInstruction(ToolMessage tool)
    {
        if (tool.Path.StartsWith("/api/files/list", StringComparison.OrdinalIgnoreCase))
            return "This is only a directory listing. It contains filenames and metadata, not file contents. If Raymond asked about the contents of a file or story, choose a path and call /api/files/read before answering.";

        if (tool.Path.StartsWith("/api/files/read", StringComparison.OrdinalIgnoreCase))
            return "This is the exact file content returned by /api/files/read. Any quote, excerpt, favorite part, or summary must be grounded in this Output. Do not invent passages or claim text appears here unless it appears in the Output.";

        if (tool.Path.StartsWith("/api/search", StringComparison.OrdinalIgnoreCase))
            return "Search results are snippets and references, not full source text. If Raymond asks for an exact quote or detailed content from a file, read the source file before answering.";

        if (tool.Path.StartsWith("/api/nightscout", StringComparison.OrdinalIgnoreCase))
            return "This is read-only Nightscout CGM data. Always mention the reading timestamp/age when using current glucose context. CGM data can lag or be inaccurate; do not infer insulin dosing, treatment, or urgent medical decisions from this alone.";

        return "Use this tool output as evidence. Do not invent facts beyond what the tool returned.";
    }

    private int EstimateMessageWithTimestampTokens(ChatMessage msg)
    {
        var ts = msg.Timestamp.ToLocalTime().ToString("dddd dd/MM/yyyy hh:mm:ss tt");
        var content = msg.Role == "assistant"
            ? ToolResultTextSanitizer.RemoveEchoedToolResultBlocks(msg.Content)
            : msg.Content;
        var text = msg.Role == "user"
            ? $"[User message written at {ts}]\n{content}"
            : content;
        return EstimateTokenCount(msg.Role) + EstimateTokenCount(text) + 6;
    }

    private string FormatMessageForCompaction(ChatMessage msg, int ordinal)
    {
        var builder = new StringBuilder();
        builder.AppendLine($"Message ordinal: {ordinal}");
        builder.AppendLine($"Message id: {msg.Id}");
        builder.AppendLine($"Role: {msg.Role}");
        builder.AppendLine($"Timestamp UTC: {msg.Timestamp.ToUniversalTime():O}");
        builder.AppendLine($"Timestamp Local: {msg.Timestamp.ToLocalTime():dddd dd/MM/yyyy hh:mm:ss tt}");
        builder.AppendLine();
        builder.AppendLine("Content:");
        builder.AppendLine(msg.Role == "assistant"
            ? ToolResultTextSanitizer.RemoveEchoedToolResultBlocks(msg.Content)
            : msg.Content);

        if (msg.Tool != null)
        {
            builder.AppendLine();
            builder.AppendLine("Tool details:");
            builder.AppendLine($"Round: {msg.Tool.Round}");
            builder.AppendLine($"Method: {msg.Tool.Method}");
            builder.AppendLine($"Path: {msg.Tool.Path}");
            builder.AppendLine($"Status: {msg.Tool.Status}");
            builder.AppendLine("Parameters:");
            builder.AppendLine(msg.Tool.Body);
            builder.AppendLine("Result:");
            builder.AppendLine(msg.Tool.Result);
        }

        if (!string.IsNullOrWhiteSpace(msg.Thinking))
        {
            builder.AppendLine();
            builder.AppendLine("Thinking:");
            builder.AppendLine(msg.Thinking);
        }

        return builder.ToString();
    }

    private CompactionConfig Settings() => _config.Compaction ?? new CompactionConfig();

    private int GetContextTokenLimit()
    {
        var settings = Settings();
        return settings.ContextTokenLimit > 0
            ? settings.ContextTokenLimit
            : _config.GetModelInputTokenLimit();
    }

    private int GetRecentContextTargetTokenCount()
    {
        var settings = Settings();
        if (settings.RecentContextTargetTokens > 0)
            return settings.RecentContextTargetTokens;

        var percent = Math.Clamp(settings.RecentContextPercent, 1, 90);
        return Math.Max(1_000, (int)Math.Floor(GetContextTokenLimit() * (percent / 100.0)));
    }

    private int GetChunkTokenLimit()
    {
        var settings = Settings();
        if (settings.ChunkTokenLimit > 0)
            return Math.Max(1_000, settings.ChunkTokenLimit);

        return Math.Max(1_000, (int)Math.Floor(GetContextTokenLimit() * 0.85));
    }

    private int GetMergeBatchTokenLimit()
    {
        var settings = Settings();
        return settings.MergeBatchTokenLimit > 0
            ? Math.Max(1_000, settings.MergeBatchTokenLimit)
            : GetChunkTokenLimit();
    }

    private int GetSummaryMaxTokens()
    {
        var settings = Settings();
        return settings.SummaryMaxTokens > 0
            ? Math.Max(1_000, settings.SummaryMaxTokens)
            : _config.GetModelOutputTokenLimit();
    }

    private string ToRelativePath(string fullPath)
    {
        return Path.GetRelativePath(_baseFolder, fullPath).Replace('\\', '/');
    }

    private static CompactionResult ToResult(CompactionPreview preview, bool dryRun)
    {
        return new CompactionResult
        {
            Enabled = preview.Enabled,
            Force = preview.Force,
            ShouldCompact = preview.ShouldCompact,
            Reason = preview.Reason,
            ModelInputTokens = preview.ModelInputTokens,
            ModelOutputTokens = preview.ModelOutputTokens,
            ContextTokenLimit = preview.ContextTokenLimit,
            TriggerTokens = preview.TriggerTokens,
            RecentContextTargetTokens = preview.RecentContextTargetTokens,
            ChunkTokenLimit = preview.ChunkTokenLimit,
            MergeBatchTokenLimit = preview.MergeBatchTokenLimit,
            SummaryMaxTokens = preview.SummaryMaxTokens,
            EstimatedApiTokens = preview.EstimatedApiTokens,
            EstimatedPostCompactionApiTokens = preview.EstimatedPostCompactionApiTokens,
            EstimatedMessagesTokens = preview.EstimatedMessagesTokens,
            EstimatedCompactedTokens = preview.EstimatedCompactedTokens,
            EstimatedKeptTokens = preview.EstimatedKeptTokens,
            EstimatedSummaryTokens = preview.EstimatedSummaryTokens,
            MessagesBefore = preview.MessagesBefore,
            MessagesToCompact = preview.MessagesToCompact,
            MessagesToKeep = preview.MessagesToKeep,
            ChunkCount = preview.ChunkCount,
            Chunks = preview.Chunks.Select(c => new CompactionChunkInfo
            {
                Index = c.Index,
                UnitCount = c.UnitCount,
                EstimatedTokens = c.EstimatedTokens,
                FirstMessageId = c.FirstMessageId,
                LastMessageId = c.LastMessageId
            }).ToList(),
            DryRun = dryRun
        };
    }

    private static string SanitizeFilename(string name, int maxLength)
    {
        var invalid = Path.GetInvalidFileNameChars();
        var sanitized = new string(name.Select(c => invalid.Contains(c) ? '_' : c).ToArray()).Trim();
        sanitized = string.IsNullOrWhiteSpace(sanitized) ? "conversation" : sanitized;
        if (maxLength > 0 && sanitized.Length > maxLength)
            sanitized = sanitized[..maxLength].TrimEnd();
        return sanitized;
    }

    private static string EscapeFrontMatter(string value)
    {
        return "\"" + value.Replace("\\", "\\\\").Replace("\"", "\\\"") + "\"";
    }

    private static string HashSummary(string summary)
    {
        var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(summary));
        return Convert.ToHexString(bytes)[..16].ToLowerInvariant();
    }

    private static List<string> SplitByLength(string text, int maxChars)
    {
        var parts = new List<string>();
        for (var i = 0; i < text.Length; i += maxChars)
            parts.Add(text.Substring(i, Math.Min(maxChars, text.Length - i)));
        return parts;
    }

    private sealed class CompactionPlan
    {
        public CompactionPreview Preview { get; init; } = new();
        public List<ChatMessage> ToCompact { get; init; } = new();
        public List<ChatMessage> Keep { get; init; } = new();
        public List<List<CompactionUnit>> Chunks { get; init; } = new();
    }

    private sealed record CompactionUnit(string MessageId, string Text, int EstimatedTokens);
    private sealed record CompactionTranscript(string Summary, string Thinking);
}