w4c-workflows-api/Services/Nodes/Binary/FileSystemBinaryStore.cs

using System.Buffers;
using System.Security.Cryptography;
using System.Text.Json.Nodes;
using w4c_workflows.Models.Nodes;

namespace w4c_workflows.Services.Nodes.Binary;

/// <summary>
/// Content-addressed binary store on the local filesystem. A payload is written
/// to a temp file while its SHA-256 is computed, then moved to
/// <c>&lt;root&gt;/&lt;aa&gt;/&lt;sha256-hex&gt;</c> and gets a JSON sidecar with
/// the display name, MIME type and size. Identical bytes always map to the same
/// asset id, so a payload fetched repeatedly is stored once.
///
/// Asset ids are validated strictly (<c>sha256:</c> + 64 hex chars) before any
/// path is built, so a crafted id from run history cannot traverse outside the
/// store root.
/// </summary>
public sealed class FileSystemBinaryStore : IBinaryStore
{
    private const string IdPrefix = "sha256:";
    private const int HashHexLength = 64;
    private const int CopyBufferSize = 81_920;

    /// <summary>Binary property name the HTTP node attaches a downloaded file under.</summary>
    public const string DefaultPropertyName = "data";

    /// <summary>
    /// In-flight temps younger than this are never swept, so a second replica
    /// sharing the same root (or a save that started moments ago) is not disturbed.
    /// </summary>
    private static readonly TimeSpan TempSweepGrace = TimeSpan.FromHours(6);

    private readonly BinaryStoreOptions _options;
    private readonly string _rootPath;

    public FileSystemBinaryStore(BinaryStoreOptions options)
    {
        _options = options;
        _rootPath = string.IsNullOrWhiteSpace(options.RootPath)
            ? Path.Combine(AppContext.BaseDirectory, "binary-store")
            : Path.GetFullPath(options.RootPath);

        // The store is a DI singleton, so constructing it is the process's first
        // touch of the store root — the natural place to clear temp files a killed
        // save left behind (temp files are the only unhashed artifacts in the store).
        SweepOrphanedTempFiles();
    }

    /// <summary>
    /// Deletes temp files left behind by a save that was killed mid-write: payload
    /// temps are <c>tmp-*</c> at the root, metadata temps are <c>*.tmp-*</c> beside
    /// the payload in a shard. Everything younger than <paramref name="grace"/>
    /// (default <see cref="TempSweepGrace"/>) is kept so a concurrent writer is
    /// never deleted. Best-effort; never throws.
    /// </summary>
    public void SweepOrphanedTempFiles(TimeSpan? grace = null)
    {
        if (!Directory.Exists(_rootPath))
            return;

        var cutoff = DateTime.UtcNow - (grace ?? TempSweepGrace);
        try
        {
            foreach (var path in Directory.EnumerateFiles(_rootPath, "tmp-*", SearchOption.TopDirectoryOnly))
                TryDeleteIfOlderThan(path, cutoff);

            foreach (var shard in Directory.EnumerateDirectories(_rootPath))
            {
                foreach (var path in Directory.EnumerateFiles(shard, "*.tmp-*", SearchOption.TopDirectoryOnly))
                    TryDeleteIfOlderThan(path, cutoff);
            }
        }
        catch (IOException)
        {
            // Best effort: a leftover temp file is harmless.
        }
        catch (UnauthorizedAccessException)
        {
            // Best effort: a leftover temp file is harmless.
        }
    }

    private static void TryDeleteIfOlderThan(string path, DateTime cutoffUtc)
    {
        try
        {
            if (File.Exists(path) && File.GetLastWriteTimeUtc(path) < cutoffUtc)
                File.Delete(path);
        }
        catch (IOException)
        {
            // Best effort: a leftover temp file is harmless.
        }
        catch (UnauthorizedAccessException)
        {
            // Best effort: a leftover temp file is harmless.
        }
    }

    /// <inheritdoc />
    public async Task<BinaryAttachment> SaveAsync(
        Stream content,
        string? fileName,
        string? mimeType,
        CancellationToken ct = default)
    {
        ArgumentNullException.ThrowIfNull(content);

        // Payloads are content-addressed, so a new directory holds only the temp
        // file until the real path is known; the root is created at first write.
        var root = _rootPath;
        Directory.CreateDirectory(root);
        var tempPath = Path.Combine(root, $"tmp-{Guid.NewGuid():N}");

        long size;
        string hashHex;
        try
        {
            using var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
            await using (var target = new FileStream(
                tempPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, CopyBufferSize, useAsync: true))
            {
                var buffer = ArrayPool<byte>.Shared.Rent(CopyBufferSize);
                try
                {
                    size = 0;
                    int read;
                    while ((read = await content.ReadAsync(buffer.AsMemory(0, buffer.Length), ct)) > 0)
                    {
                        size += read;
                        if (_options.MaxBytes > 0 && size > _options.MaxBytes)
                            throw new InvalidOperationException(
                                $"binary payload exceeds the {_options.MaxBytes}-byte store limit");

                        hasher.AppendData(buffer, 0, read);
                        await target.WriteAsync(buffer.AsMemory(0, read), ct);
                    }
                }
                finally
                {
                    ArrayPool<byte>.Shared.Return(buffer);
                }
            }

            hashHex = Convert.ToHexString(hasher.GetHashAndReset()).ToLowerInvariant();
        }
        catch
        {
            TryDelete(tempPath);
            throw;
        }

        var shard = Path.Combine(root, hashHex[..2]);
        Directory.CreateDirectory(shard);
        var payloadPath = Path.Combine(shard, hashHex);
        var metaPath = payloadPath + ".json";

        try
        {
            if (File.Exists(payloadPath))
            {
                // Same bytes already stored: keep the first metadata, drop the temp copy.
                TryDelete(tempPath);
            }
            else
            {
                File.Move(tempPath, payloadPath);
            }
        }
        catch (IOException)
        {
            // Lost a race with a concurrent store of the same content: the winner's
            // payload is equivalent, so discard ours.
            TryDelete(tempPath);
        }

        var attachment = new BinaryAttachment(BuildAssetId(hashHex), CleanName(fileName), CleanMime(mimeType), size);
        if (!File.Exists(metaPath))
            await WriteMetadataAsync(metaPath, attachment, ct);

        return attachment;
    }

    /// <inheritdoc />
    public Task<Stream?> OpenAsync(string assetId, CancellationToken ct = default)
    {
        ct.ThrowIfCancellationRequested();
        if (!TryResolve(assetId, out var payloadPath) || !File.Exists(payloadPath))
            return Task.FromResult<Stream?>(null);

        Stream stream = new FileStream(
            payloadPath, FileMode.Open, FileAccess.Read, FileShare.Read, CopyBufferSize, useAsync: true);
        return Task.FromResult<Stream?>(stream);
    }

    /// <inheritdoc />
    public async Task<BinaryAttachment?> DescribeAsync(string assetId, CancellationToken ct = default)
    {
        if (!TryResolve(assetId, out var payloadPath))
            return null;

        var metaPath = payloadPath + ".json";
        if (!File.Exists(metaPath))
            return File.Exists(payloadPath)
                ? new BinaryAttachment(assetId, SizeBytes: new FileInfo(payloadPath).Length)
                : null;

        try
        {
            var text = await File.ReadAllTextAsync(metaPath, ct);
            if (JsonNode.Parse(text) is not JsonObject meta)
                return null;

            return new BinaryAttachment(
                assetId,
                meta["fileName"]?.GetValue<string>(),
                meta["mimeType"]?.GetValue<string>(),
                meta["sizeBytes"]?.GetValue<long>());
        }
        catch (Exception)
        {
            // A corrupt sidecar must not fail the caller; fall back to the payload.
            return new BinaryAttachment(assetId, SizeBytes: new FileInfo(payloadPath).Length);
        }
    }

    /// <summary>Builds the canonical asset id for a computed SHA-256 hash.</summary>
    public static string BuildAssetId(string hashHex) => IdPrefix + hashHex;

    /// <summary>Validates an asset id and maps it to its payload path (no traversal).</summary>
    private bool TryResolve(string assetId, out string payloadPath)
    {
        payloadPath = string.Empty;
        if (string.IsNullOrWhiteSpace(assetId) || !assetId.StartsWith(IdPrefix, StringComparison.Ordinal))
            return false;

        var hex = assetId[IdPrefix.Length..];
        if (hex.Length != HashHexLength)
            return false;

        foreach (var character in hex)
        {
            var lower = char.ToLowerInvariant(character);
            if ((lower < '0' || lower > '9') && (lower < 'a' || lower > 'f'))
                return false;
        }

        hex = hex.ToLowerInvariant();
        payloadPath = Path.Combine(_rootPath, hex[..2], hex);
        return true;
    }

    private static async Task WriteMetadataAsync(string metaPath, BinaryAttachment attachment, CancellationToken ct)
    {
        var meta = new JsonObject
        {
            ["fileName"] = attachment.FileName,
            ["mimeType"] = attachment.MimeType,
            ["sizeBytes"] = attachment.SizeBytes,
        };

        var tempPath = metaPath + $".tmp-{Guid.NewGuid():N}";
        try
        {
            await File.WriteAllTextAsync(tempPath, meta.ToJsonString(), ct);
            if (File.Exists(metaPath))
                File.Delete(tempPath);
            else
                File.Move(tempPath, metaPath);
        }
        catch (IOException)
        {
            TryDelete(tempPath);
        }
        catch (OperationCanceledException)
        {
            TryDelete(tempPath);
            throw;
        }
    }

    private static string? CleanName(string? fileName)
        => string.IsNullOrWhiteSpace(fileName) ? null : fileName.Trim();

    private static string? CleanMime(string? mimeType)
        => string.IsNullOrWhiteSpace(mimeType) ? null : mimeType.Trim();

    private static void TryDelete(string path)
    {
        try
        {
            if (File.Exists(path))
                File.Delete(path);
        }
        catch (IOException)
        {
            // Best effort: a leftover temp file is harmless.
        }
        catch (UnauthorizedAccessException)
        {
            // Best effort: a leftover temp file is harmless.
        }
    }
}