Add workflow dispatch transactional outbox (#7517)
* Add workflow dispatch transactional outbox * Address workflow dispatch outbox review * Make outbox state lookup null-safe * Generate stable outbox dispatch instance ids * Limit eager outbox processing to marked commits * Avoid blocking eager outbox processing on lock contention * Expire uncommitted outbox items after retention * Harden eager workflow outbox processing * Prune delivered workflow outbox markers * Batch workflow dispatch outbox processing * Bound workflow dispatch outbox processing batches * Harden workflow dispatch outbox processing * Address transactional outbox review feedback * Address transactional outbox follow-up review * Address outbox recovery review feedback * Address outbox key-value review fixes * Address outbox cleanup review feedback * Address workflow outbox review feedback * Address outbox key-value review feedback * Preserve outbox key-value tenant ids * Address workflow outbox marker cleanup review * Ensure key value store factory registrations override * Address workflow outbox review follow-ups
This commit is contained in:
Sipke Schoorstra
GitHub
parent
541218a37f
commit
d09a1b7cb4
|
|
@ -59,6 +59,7 @@ Key files:
|
|||
|
||||
- [LocalWorkflowRuntime](../../src/modules/Elsa.Workflows.Runtime/Services/LocalWorkflowRuntime.cs)
|
||||
- [BackgroundWorkflowDispatcher](../../src/modules/Elsa.Workflows.Runtime/Services/BackgroundWorkflowDispatcher.cs)
|
||||
- [TransactionalWorkflowDispatcher](../../src/modules/Elsa.Workflows.Runtime/Services/TransactionalWorkflowDispatcher.cs)
|
||||
- [ValidatingWorkflowDispatcher](../../src/modules/Elsa.Workflows.Runtime/Services/ValidatingWorkflowDispatcher.cs)
|
||||
- [WorkflowInvoker](../../src/modules/Elsa.Workflows.Runtime/Services/WorkflowInvoker.cs)
|
||||
- [DefaultWorkflowStarter](../../src/modules/Elsa.Workflows.Runtime/Services/DefaultWorkflowStarter.cs)
|
||||
|
|
@ -66,6 +67,27 @@ Key files:
|
|||
- [BookmarkResumer](../../src/modules/Elsa.Workflows.Runtime/Services/BookmarkResumer.cs)
|
||||
- [TriggerInvoker](../../src/modules/Elsa.Workflows.Runtime/Services/TriggerInvoker.cs)
|
||||
|
||||
## Transactional Dispatch Outbox
|
||||
|
||||
Hosts can opt into at-least-once workflow dispatch for dispatch calls made from inside a running workflow:
|
||||
|
||||
```csharp
|
||||
services.Configure<WorkflowDispatcherOptions>(options =>
|
||||
{
|
||||
options.UseTransactionalOutbox = true;
|
||||
});
|
||||
```
|
||||
|
||||
When enabled, [TransactionalWorkflowDispatcher](../../src/modules/Elsa.Workflows.Runtime/Services/TransactionalWorkflowDispatcher.cs) writes the command to [IWorkflowDispatchOutboxStore](../../src/modules/Elsa.Workflows.Runtime/Contracts/IWorkflowDispatchOutboxStore.cs) before the parent workflow state commits, and stores the outbox item ID in the parent `WorkflowState.Properties`. [WorkflowDispatchOutboxProcessor](../../src/modules/Elsa.Workflows.Runtime/Services/WorkflowDispatchOutboxProcessor.cs) delivers only records whose owner workflow state contains that committed marker. This prevents a crash between workflow-state commit and mediator enqueue from silently losing the dispatch: the durable outbox record is already present, and the committed marker authorizes delivery after restart.
|
||||
|
||||
Operational notes:
|
||||
|
||||
- The default outbox store uses `IKeyValueStore`; production hosts should pair this option with durable workflow instance persistence and durable key-value persistence.
|
||||
- Delivery is at-least-once. If the process crashes after sending a command but before deleting the outbox record, the processor may send it again.
|
||||
- Workflow definition dispatches generated by `DispatchWorkflow`/`BulkDispatchWorkflows` include a child workflow instance ID. [DispatchWorkflowRequestHandler](../../src/modules/Elsa.Workflows.Runtime/Handlers/DispatchWorkflowRequestHandler.cs) treats that ID as the idempotency key for outbox-routed commands and skips duplicate create-and-run attempts when the instance already exists.
|
||||
- Outbox processing is serialized with the configured distributed lock provider. Poison items are abandoned after `WorkflowDispatcherOptions.MaxOutboxDeliveryAttempts`, and missing-owner items are removed after `WorkflowDispatcherOptions.OrphanedOutboxItemRetention`.
|
||||
- Dispatch calls outside a workflow execution context continue to use the regular background dispatcher.
|
||||
|
||||
## Triggers And Bookmarks
|
||||
|
||||
Triggers start workflows. Bookmarks resume suspended workflow instances. Runtime indexes and queries them through:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
using Elsa.Common.Services;
|
||||
using Elsa.Features.Abstractions;
|
||||
using Elsa.Features.Services;
|
||||
using Elsa.KeyValues.Contracts;
|
||||
using Elsa.KeyValues.Entities;
|
||||
using Elsa.KeyValues.Stores;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace Elsa.KeyValues.Features;
|
||||
|
||||
|
|
@ -11,6 +14,10 @@ namespace Elsa.KeyValues.Features;
|
|||
/// </summary>
|
||||
public class KeyValueFeature : FeatureBase
|
||||
{
|
||||
private static readonly Func<IServiceProvider, IKeyValueStore> DefaultKeyValueStore = sp => ActivatorUtilities.CreateInstance<MemoryKeyValueStore>(sp);
|
||||
private Func<IServiceProvider, IKeyValueStore> _keyValueStore = DefaultKeyValueStore;
|
||||
private bool? _registerMemoryStore;
|
||||
|
||||
/// <inheritdoc />
|
||||
public KeyValueFeature(IModule module) : base(module)
|
||||
{
|
||||
|
|
@ -19,12 +26,27 @@ public class KeyValueFeature : FeatureBase
|
|||
/// <summary>
|
||||
/// A factory that instantiates an <see cref="IKeyValueStore"/>.
|
||||
/// </summary>
|
||||
public Func<IServiceProvider, IKeyValueStore> KeyValueStore { get; set; } = sp => ActivatorUtilities.CreateInstance<MemoryKeyValueStore>(sp);
|
||||
public Func<IServiceProvider, IKeyValueStore> KeyValueStore
|
||||
{
|
||||
get => _keyValueStore;
|
||||
set => _keyValueStore = value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether to register the in-memory backing store used by <see cref="MemoryKeyValueStore"/>.
|
||||
/// </summary>
|
||||
public bool RegisterMemoryStore
|
||||
{
|
||||
get => _registerMemoryStore ?? ReferenceEquals(KeyValueStore, DefaultKeyValueStore);
|
||||
set => _registerMemoryStore = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void Apply()
|
||||
{
|
||||
Services.AddScoped(KeyValueStore);
|
||||
if (RegisterMemoryStore)
|
||||
Services.TryAddSingleton<MemoryStore<SerializedKeyValuePair>>();
|
||||
|
||||
Services.AddScoped<IKeyValueStore>(KeyValueStore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,17 @@ public class KeyValueFilter
|
|||
/// </summary>
|
||||
public ICollection<string>? Keys { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum number of records to return. <c>null</c> means no limit; <c>0</c> returns no records.
|
||||
/// </summary>
|
||||
public int? Take { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether results should be ordered by the persisted key before applying <see cref="Take"/>.
|
||||
/// The persisted key is stored as <c>Id</c>; <see cref="SerializedKeyValuePair.Key"/> is an alias.
|
||||
/// </summary>
|
||||
public bool OrderByKey { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Applies the filter to the specified queryable.
|
||||
/// </summary>
|
||||
|
|
@ -36,6 +47,12 @@ public class KeyValueFilter
|
|||
|
||||
if (filter.Keys != null) queryable = queryable.Where(x => filter.Keys.Contains(x.Id));
|
||||
|
||||
if (filter.OrderByKey)
|
||||
queryable = queryable.OrderBy(x => x.Id);
|
||||
|
||||
if (filter.Take != null)
|
||||
queryable = queryable.Take(Math.Max(0, filter.Take.Value));
|
||||
|
||||
return queryable;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
using CShells.Features;
|
||||
using Elsa.Common.Services;
|
||||
using Elsa.KeyValues.Contracts;
|
||||
using Elsa.KeyValues.Entities;
|
||||
using Elsa.KeyValues.Stores;
|
||||
using JetBrains.Annotations;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace Elsa.KeyValues.ShellFeatures;
|
||||
|
||||
|
|
@ -15,13 +18,33 @@ namespace Elsa.KeyValues.ShellFeatures;
|
|||
[UsedImplicitly]
|
||||
public class KeyValueFeature : IShellFeature
|
||||
{
|
||||
private static readonly Func<IServiceProvider, IKeyValueStore> DefaultKeyValueStore = sp => ActivatorUtilities.CreateInstance<MemoryKeyValueStore>(sp);
|
||||
private Func<IServiceProvider, IKeyValueStore> _keyValueStore = DefaultKeyValueStore;
|
||||
private bool? _registerMemoryStore;
|
||||
|
||||
/// <summary>
|
||||
/// A factory that instantiates an <see cref="IKeyValueStore"/>.
|
||||
/// </summary>
|
||||
public Func<IServiceProvider, IKeyValueStore> KeyValueStore { get; set; } = sp => ActivatorUtilities.CreateInstance<MemoryKeyValueStore>(sp);
|
||||
public Func<IServiceProvider, IKeyValueStore> KeyValueStore
|
||||
{
|
||||
get => _keyValueStore;
|
||||
set => _keyValueStore = value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether to register the in-memory backing store used by <see cref="MemoryKeyValueStore"/>.
|
||||
/// </summary>
|
||||
public bool RegisterMemoryStore
|
||||
{
|
||||
get => _registerMemoryStore ?? ReferenceEquals(KeyValueStore, DefaultKeyValueStore);
|
||||
set => _registerMemoryStore = value;
|
||||
}
|
||||
|
||||
public void ConfigureServices(IServiceCollection services)
|
||||
{
|
||||
services.AddScoped(KeyValueStore);
|
||||
if (RegisterMemoryStore)
|
||||
services.TryAddSingleton<MemoryStore<SerializedKeyValuePair>>();
|
||||
|
||||
services.AddScoped<IKeyValueStore>(KeyValueStore);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
3
src/modules/Elsa.Workflows.Runtime/AssemblyInfo.cs
Normal file
3
src/modules/Elsa.Workflows.Runtime/AssemblyInfo.cs
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("Elsa.Workflows.Runtime.UnitTests")]
|
||||
|
|
@ -18,4 +18,9 @@ public class DispatchWorkflowDefinitionCommand(string definitionVersionId) : ICo
|
|||
public string? SchedulingActivityExecutionId { get; set; }
|
||||
public string? SchedulingWorkflowInstanceId { get; set; }
|
||||
public int? SchedulingCallStackDepth { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When set, creating the workflow instance is skipped if <see cref="InstanceId"/> already exists.
|
||||
/// </summary>
|
||||
public bool SkipIfInstanceExists { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
using Elsa.Workflows.Runtime.Models;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Stores workflow dispatch commands for at-least-once delivery after their owner workflow state commits.
|
||||
/// </summary>
|
||||
public interface IWorkflowDispatchOutbox
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds the specified dispatch item to the durable outbox and marks the current workflow state as owning it.
|
||||
/// </summary>
|
||||
Task EnqueueAsync(WorkflowExecutionContext workflowExecutionContext, WorkflowDispatchOutboxItem item, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Provides access to the workflow execution context currently running on the async call path.
|
||||
/// </summary>
|
||||
public interface IWorkflowDispatchOutboxAccessor
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the current workflow execution context.
|
||||
/// </summary>
|
||||
WorkflowExecutionContext? WorkflowExecutionContext { get; set; }
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Processes pending workflow dispatch outbox items.
|
||||
/// </summary>
|
||||
public interface IWorkflowDispatchOutboxProcessor
|
||||
{
|
||||
/// <summary>
|
||||
/// Processes pending outbox items.
|
||||
/// </summary>
|
||||
Task ProcessAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to process pending outbox items without waiting for the processor lock.
|
||||
/// </summary>
|
||||
Task<bool> TryProcessAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
using Elsa.Workflows.Runtime.Models;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Durable store for workflow dispatch outbox items.
|
||||
/// </summary>
|
||||
public interface IWorkflowDispatchOutboxStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Saves the specified outbox item.
|
||||
/// </summary>
|
||||
Task SaveAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Returns all pending outbox items.
|
||||
/// </summary>
|
||||
Task<IEnumerable<WorkflowDispatchOutboxItem>> FindManyAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Returns up to the specified number of pending outbox items.
|
||||
/// </summary>
|
||||
/// <param name="maxCount">The maximum number of pending items to return. Values less than or equal to 0 disable the limit.</param>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
Task<IEnumerable<WorkflowDispatchOutboxItem>> FindManyAsync(int maxCount, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes the specified outbox item.
|
||||
/// </summary>
|
||||
Task DeleteAsync(string id, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
using Elsa.Workflows.Runtime.Commands;
|
||||
using Elsa.Workflows.Runtime.Requests;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
internal static class WorkflowDispatchCommandFactory
|
||||
{
|
||||
public static DispatchWorkflowDefinitionCommand CreateCommand(DispatchWorkflowDefinitionRequest request, string? instanceId = null)
|
||||
{
|
||||
var useGeneratedInstanceId = string.IsNullOrWhiteSpace(request.InstanceId);
|
||||
|
||||
return new(request.DefinitionVersionId)
|
||||
{
|
||||
Input = request.Input,
|
||||
Properties = request.Properties,
|
||||
CorrelationId = request.CorrelationId,
|
||||
InstanceId = useGeneratedInstanceId ? instanceId : request.InstanceId,
|
||||
TriggerActivityId = request.TriggerActivityId,
|
||||
ParentWorkflowInstanceId = request.ParentWorkflowInstanceId,
|
||||
SchedulingActivityExecutionId = request.SchedulingActivityExecutionId,
|
||||
SchedulingWorkflowInstanceId = request.SchedulingWorkflowInstanceId,
|
||||
SchedulingCallStackDepth = request.SchedulingCallStackDepth,
|
||||
SkipIfInstanceExists = useGeneratedInstanceId && !string.IsNullOrWhiteSpace(instanceId)
|
||||
};
|
||||
}
|
||||
|
||||
public static DispatchWorkflowInstanceCommand CreateCommand(DispatchWorkflowInstanceRequest request)
|
||||
{
|
||||
return new(request.InstanceId)
|
||||
{
|
||||
BookmarkId = request.BookmarkId,
|
||||
ActivityHandle = request.ActivityHandle,
|
||||
Input = request.Input,
|
||||
Properties = request.Properties,
|
||||
CorrelationId = request.CorrelationId
|
||||
};
|
||||
}
|
||||
|
||||
public static DispatchTriggerWorkflowsCommand CreateCommand(DispatchTriggerWorkflowsRequest request)
|
||||
{
|
||||
return new(request.ActivityTypeName, request.BookmarkPayload)
|
||||
{
|
||||
CorrelationId = request.CorrelationId,
|
||||
WorkflowInstanceId = request.WorkflowInstanceId,
|
||||
ActivityInstanceId = request.ActivityInstanceId,
|
||||
Input = request.Input,
|
||||
Properties = request.Properties
|
||||
};
|
||||
}
|
||||
|
||||
public static DispatchResumeWorkflowsCommand CreateCommand(DispatchResumeWorkflowsRequest request)
|
||||
{
|
||||
return new(request.ActivityTypeName, request.BookmarkPayload)
|
||||
{
|
||||
CorrelationId = request.CorrelationId,
|
||||
WorkflowInstanceId = request.WorkflowInstanceId,
|
||||
ActivityInstanceId = request.ActivityInstanceId,
|
||||
Input = request.Input
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
using Elsa.Extensions;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
using Elsa.Workflows.State;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for workflow dispatch outbox state stored on workflow state.
|
||||
/// </summary>
|
||||
public static class WorkflowDispatchOutboxStateExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// The workflow property key used to store committed outbox item IDs.
|
||||
/// </summary>
|
||||
public const string PropertyKey = "Elsa.Workflows.Runtime.WorkflowDispatchOutbox";
|
||||
|
||||
/// <summary>
|
||||
/// Adds an outbox item ID to the workflow execution context's persisted property bag.
|
||||
/// </summary>
|
||||
public static void AddWorkflowDispatchOutboxItem(this WorkflowExecutionContext context, string outboxItemId)
|
||||
{
|
||||
var state = context.Properties.GetWorkflowDispatchOutboxState();
|
||||
|
||||
if (!state.ItemIds.Contains(outboxItemId))
|
||||
state.ItemIds.Add(outboxItemId);
|
||||
|
||||
context.Properties[PropertyKey] = state;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a delivered outbox item ID from the persisted workflow state property bag.
|
||||
/// </summary>
|
||||
public static bool RemoveWorkflowDispatchOutboxItem(this WorkflowState workflowState, string outboxItemId)
|
||||
{
|
||||
var state = workflowState.Properties.GetWorkflowDispatchOutboxState();
|
||||
|
||||
if (!state.ItemIds.Remove(outboxItemId))
|
||||
return false;
|
||||
|
||||
if (state.ItemIds.Count == 0)
|
||||
workflowState.Properties.Remove(PropertyKey);
|
||||
else
|
||||
workflowState.Properties[PropertyKey] = state;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true when the workflow state includes the specified committed outbox item ID.
|
||||
/// </summary>
|
||||
public static bool HasWorkflowDispatchOutboxItem(this WorkflowState? workflowState, string outboxItemId)
|
||||
{
|
||||
return workflowState?.Properties.GetWorkflowDispatchOutboxState().ItemIds.Contains(outboxItemId) == true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true when the workflow state includes committed outbox item IDs.
|
||||
/// </summary>
|
||||
public static bool HasWorkflowDispatchOutboxItems(this WorkflowState? workflowState)
|
||||
{
|
||||
return workflowState?.Properties.GetWorkflowDispatchOutboxState().ItemIds.Count > 0;
|
||||
}
|
||||
|
||||
private static WorkflowDispatchOutboxState GetWorkflowDispatchOutboxState(this IDictionary<string, object>? properties)
|
||||
{
|
||||
return properties?.TryGetValue<WorkflowDispatchOutboxState>(PropertyKey, out var state) == true ? state : new();
|
||||
}
|
||||
}
|
||||
|
|
@ -18,6 +18,7 @@ public static class WorkflowExecutionPipelineBuilderExtensions
|
|||
pipelineBuilder
|
||||
.Reset()
|
||||
.UseExecutionCycleTracking()
|
||||
.UseWorkflowDispatchOutbox()
|
||||
.UseWorkflowHeartbeat()
|
||||
.UseEngineExceptionHandling()
|
||||
.UsePersistentVariables()
|
||||
|
|
@ -38,6 +39,11 @@ public static class WorkflowExecutionPipelineBuilderExtensions
|
|||
/// </summary>
|
||||
public static IWorkflowExecutionPipelineBuilder UseExecutionCycleTracking(this IWorkflowExecutionPipelineBuilder pipelineBuilder) => pipelineBuilder.UseMiddleware<ExecutionCycleTrackingMiddleware>();
|
||||
|
||||
/// <summary>
|
||||
/// Installs middleware that scopes the current workflow execution context for transactional workflow dispatch outbox writes.
|
||||
/// </summary>
|
||||
public static IWorkflowExecutionPipelineBuilder UseWorkflowDispatchOutbox(this IWorkflowExecutionPipelineBuilder pipelineBuilder) => pipelineBuilder.UseMiddleware<WorkflowDispatchOutboxMiddleware>();
|
||||
|
||||
/// <summary>
|
||||
/// Installs middleware that persists bookmarks after workflow execution.
|
||||
/// </summary>
|
||||
|
|
@ -55,4 +61,4 @@ public static class WorkflowExecutionPipelineBuilderExtensions
|
|||
/// </summary>
|
||||
[Obsolete("This middleware is no longer used and will be removed in a future version. Activity state is now persisted through the commit state handler.")]
|
||||
public static IWorkflowExecutionPipelineBuilder UseActivityExecutionLogPersistence(this IWorkflowExecutionPipelineBuilder pipelineBuilder) => pipelineBuilder.UseMiddleware<PersistActivityExecutionLogMiddleware>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ using Elsa.Workflows.Runtime.UIHints;
|
|||
using Medallion.Threading;
|
||||
using Medallion.Threading.FileSystem;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.Features;
|
||||
|
|
@ -52,7 +53,8 @@ public class WorkflowRuntimeFeature(IModule module) : FeatureBase(module)
|
|||
public Func<IServiceProvider, IWorkflowDispatcher> WorkflowDispatcher { get; set; } = sp =>
|
||||
{
|
||||
var decoratedService = ActivatorUtilities.CreateInstance<BackgroundWorkflowDispatcher>(sp);
|
||||
return ActivatorUtilities.CreateInstance<ValidatingWorkflowDispatcher>(sp, decoratedService);
|
||||
var transactionalService = ActivatorUtilities.CreateInstance<TransactionalWorkflowDispatcher>(sp, decoratedService);
|
||||
return ActivatorUtilities.CreateInstance<ValidatingWorkflowDispatcher>(sp, transactionalService);
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -65,6 +67,11 @@ public class WorkflowRuntimeFeature(IModule module) : FeatureBase(module)
|
|||
/// </summary>
|
||||
public Func<IServiceProvider, IWorkflowCancellationDispatcher> WorkflowCancellationDispatcher { get; set; } = sp => ActivatorUtilities.CreateInstance<BackgroundWorkflowCancellationDispatcher>(sp);
|
||||
|
||||
/// <summary>
|
||||
/// A factory that instantiates an <see cref="IWorkflowDispatchOutboxStore"/>.
|
||||
/// </summary>
|
||||
public Func<IServiceProvider, IWorkflowDispatchOutboxStore> WorkflowDispatchOutboxStore { get; set; } = sp => ActivatorUtilities.CreateInstance<KeyValueWorkflowDispatchOutboxStore>(sp);
|
||||
|
||||
/// <summary>
|
||||
/// A factory that instantiates an <see cref="IBookmarkStore"/>.
|
||||
/// </summary>
|
||||
|
|
@ -340,6 +347,7 @@ public class WorkflowRuntimeFeature(IModule module) : FeatureBase(module)
|
|||
.AddScoped<IWorkflowStarter, DefaultWorkflowStarter>()
|
||||
.AddScoped<IWorkflowRestarter, DefaultWorkflowRestarter>()
|
||||
.AddScoped<IBookmarkQueuePurger, DefaultBookmarkQueuePurger>()
|
||||
.AddSingleton<IWorkflowDispatchOutboxAccessor, WorkflowDispatchOutboxAccessor>()
|
||||
.AddScoped<ILogRecordExtractor<WorkflowExecutionLogRecord>, WorkflowExecutionLogRecordExtractor>()
|
||||
.AddScoped<IActivityPropertyLogPersistenceEvaluator, ActivityPropertyLogPersistenceEvaluator>()
|
||||
.AddScoped<IBookmarkQueueProcessor, BookmarkQueueProcessor>()
|
||||
|
|
@ -379,6 +387,7 @@ public class WorkflowRuntimeFeature(IModule module) : FeatureBase(module)
|
|||
.AddRecurringTask<TriggerBookmarkQueueRecurringTask>(TimeSpan.FromMinutes(1))
|
||||
.AddRecurringTask<PurgeBookmarkQueueRecurringTask>(TimeSpan.FromSeconds(10))
|
||||
.AddRecurringTask<RestartInterruptedWorkflowsTask>(TimeSpan.FromMinutes(5)) // Same default as the workflow liveness threshold.
|
||||
.AddRecurringTask<ProcessWorkflowDispatchOutboxRecurringTask>(TimeSpan.FromSeconds(10))
|
||||
|
||||
// Distributed locking.
|
||||
.AddSingleton(DistributedLockProvider)
|
||||
|
|
@ -395,6 +404,7 @@ public class WorkflowRuntimeFeature(IModule module) : FeatureBase(module)
|
|||
.AddCommandHandler<CancelWorkflowsCommandHandler>()
|
||||
.AddNotificationHandler<ResumeDispatchWorkflowActivity>()
|
||||
.AddNotificationHandler<ResumeBulkDispatchWorkflowActivity>()
|
||||
.AddNotificationHandler<ProcessWorkflowDispatchOutbox>()
|
||||
.AddNotificationHandler<ResumeExecuteWorkflowActivity>()
|
||||
.AddNotificationHandler<IndexTriggers>()
|
||||
.AddNotificationHandler<CancelBackgroundActivities>()
|
||||
|
|
@ -413,5 +423,9 @@ public class WorkflowRuntimeFeature(IModule module) : FeatureBase(module)
|
|||
.AddScoped<IWorkflowActivationStrategy, CorrelatedSingletonStrategy>()
|
||||
.AddScoped<IWorkflowActivationStrategy, CorrelationStrategy>()
|
||||
;
|
||||
|
||||
Services.TryAddScoped<IWorkflowDispatchOutbox>(sp => ActivatorUtilities.CreateInstance<WorkflowDispatchOutbox>(sp));
|
||||
Services.TryAddScoped(WorkflowDispatchOutboxStore);
|
||||
Services.TryAddScoped<IWorkflowDispatchOutboxProcessor, WorkflowDispatchOutboxProcessor>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,10 @@ internal class DispatchWorkflowCommandHandler(IStimulusSender stimulusSender, IW
|
|||
public virtual async Task<Unit> HandleAsync(DispatchWorkflowDefinitionCommand command, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = await workflowRuntime.CreateClientAsync(command.InstanceId, cancellationToken);
|
||||
|
||||
if (command.SkipIfInstanceExists && !string.IsNullOrWhiteSpace(command.InstanceId) && await client.InstanceExistsAsync(cancellationToken))
|
||||
return Unit.Instance;
|
||||
|
||||
var createRequest = new CreateAndRunWorkflowInstanceRequest
|
||||
{
|
||||
WorkflowDefinitionHandle = WorkflowDefinitionHandle.ByDefinitionVersionId(command.DefinitionVersionId),
|
||||
|
|
@ -82,4 +86,4 @@ internal class DispatchWorkflowCommandHandler(IStimulusSender stimulusSender, IW
|
|||
|
||||
return Unit.Instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
using Elsa.Mediator.Contracts;
|
||||
using Elsa.Workflows.Runtime.Notifications;
|
||||
using Elsa.Workflows.Runtime.Options;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.Handlers;
|
||||
|
||||
/// <summary>
|
||||
/// Processes the workflow dispatch outbox after a workflow state commit.
|
||||
/// </summary>
|
||||
public class ProcessWorkflowDispatchOutbox(IServiceProvider serviceProvider, IOptions<WorkflowDispatcherOptions> options, ILogger<ProcessWorkflowDispatchOutbox> logger) : INotificationHandler<WorkflowStateCommitted>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async Task HandleAsync(WorkflowStateCommitted notification, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Value.UseTransactionalOutbox || !options.Value.ProcessOutboxAfterCommit)
|
||||
return;
|
||||
|
||||
if (!notification.WorkflowState.HasWorkflowDispatchOutboxItems())
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
var processor = serviceProvider.GetRequiredService<IWorkflowDispatchOutboxProcessor>();
|
||||
await processor.TryProcessAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(e, "Failed to process workflow dispatch outbox after workflow state commit; the recurring outbox sweep will retry pending items.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
using Elsa.Workflows.Pipelines.WorkflowExecution;
|
||||
using Elsa.Workflows.Runtime.Options;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.Middleware.Workflows;
|
||||
|
||||
/// <summary>
|
||||
/// Makes the current workflow execution context available to the workflow dispatch outbox.
|
||||
/// </summary>
|
||||
public class WorkflowDispatchOutboxMiddleware(
|
||||
WorkflowMiddlewareDelegate next,
|
||||
IWorkflowDispatchOutboxAccessor accessor,
|
||||
IOptions<WorkflowDispatcherOptions> options) : WorkflowExecutionMiddleware(next)
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask InvokeAsync(WorkflowExecutionContext context)
|
||||
{
|
||||
if (!options.Value.UseTransactionalOutbox)
|
||||
{
|
||||
await Next(context);
|
||||
return;
|
||||
}
|
||||
|
||||
var previousContext = accessor.WorkflowExecutionContext;
|
||||
accessor.WorkflowExecutionContext = context;
|
||||
|
||||
try
|
||||
{
|
||||
await Next(context);
|
||||
}
|
||||
finally
|
||||
{
|
||||
accessor.WorkflowExecutionContext = previousContext;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
using Elsa.Workflows.Runtime.Commands;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.Models;
|
||||
|
||||
/// <summary>
|
||||
/// A durable workflow dispatch command waiting for delivery.
|
||||
/// </summary>
|
||||
public class WorkflowDispatchOutboxItem
|
||||
{
|
||||
/// <summary>
|
||||
/// The outbox item ID.
|
||||
/// </summary>
|
||||
public string Id { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// The workflow instance whose committed state must reference this item before it can be delivered.
|
||||
/// </summary>
|
||||
public string OwnerWorkflowInstanceId { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// The tenant ID captured when the item was enqueued.
|
||||
/// </summary>
|
||||
public string? TenantId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The kind of dispatch command stored by this item.
|
||||
/// </summary>
|
||||
public WorkflowDispatchOutboxItemKind Kind { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the item was created.
|
||||
/// </summary>
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The number of failed delivery attempts.
|
||||
/// </summary>
|
||||
public int DeliveryAttempts { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A workflow definition dispatch command.
|
||||
/// </summary>
|
||||
public DispatchWorkflowDefinitionCommand? WorkflowDefinitionCommand { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A workflow instance dispatch command.
|
||||
/// </summary>
|
||||
public DispatchWorkflowInstanceCommand? WorkflowInstanceCommand { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A trigger workflow dispatch command.
|
||||
/// </summary>
|
||||
public DispatchTriggerWorkflowsCommand? TriggerWorkflowsCommand { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A resume workflows dispatch command.
|
||||
/// </summary>
|
||||
public DispatchResumeWorkflowsCommand? ResumeWorkflowsCommand { get; set; }
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
namespace Elsa.Workflows.Runtime.Models;
|
||||
|
||||
/// <summary>
|
||||
/// The supported workflow dispatch outbox command kinds.
|
||||
/// </summary>
|
||||
public enum WorkflowDispatchOutboxItemKind
|
||||
{
|
||||
/// <summary>
|
||||
/// Dispatches a workflow definition.
|
||||
/// </summary>
|
||||
WorkflowDefinition,
|
||||
|
||||
/// <summary>
|
||||
/// Dispatches an existing workflow instance.
|
||||
/// </summary>
|
||||
WorkflowInstance,
|
||||
|
||||
/// <summary>
|
||||
/// Dispatches workflows by trigger stimulus.
|
||||
/// </summary>
|
||||
TriggerWorkflows,
|
||||
|
||||
/// <summary>
|
||||
/// Dispatches workflows by resume stimulus.
|
||||
/// </summary>
|
||||
ResumeWorkflows
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
namespace Elsa.Workflows.Runtime.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Workflow-state marker proving that outbox items belong to a committed workflow state.
|
||||
/// </summary>
|
||||
public class WorkflowDispatchOutboxState
|
||||
{
|
||||
/// <summary>
|
||||
/// Outbox item IDs committed with the workflow state.
|
||||
/// </summary>
|
||||
public ICollection<string> ItemIds { get; set; } = new List<string>();
|
||||
}
|
||||
|
|
@ -12,4 +12,32 @@ public class WorkflowDispatcherOptions
|
|||
/// Channels are used to dispatch workflows to different queues or endpoints.
|
||||
/// </remarks>
|
||||
public ICollection<DispatcherChannel> Channels { get; set; } = new List<DispatcherChannel>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether workflow dispatch calls made during workflow execution are written to the transactional outbox.
|
||||
/// </summary>
|
||||
public bool UseTransactionalOutbox { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether the shared outbox processor should run immediately after a workflow state commit that contains outbox items.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The immediate processor drains eligible pending items from the shared outbox. Disable this option to rely only on the recurring outbox sweep when commit latency is more important than eager dispatch.
|
||||
/// </remarks>
|
||||
public bool ProcessOutboxAfterCommit { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets how long an outbox item whose owner workflow instance cannot be found should be retained before cleanup.
|
||||
/// </summary>
|
||||
public TimeSpan OrphanedOutboxItemRetention { get; set; } = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum number of failed delivery attempts before an outbox item is abandoned.
|
||||
/// </summary>
|
||||
public int MaxOutboxDeliveryAttempts { get; set; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum number of outbox items to load and process during one processor cycle.
|
||||
/// </summary>
|
||||
public int OutboxProcessorBatchSize { get; set; } = 100;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,19 +19,7 @@ public class BackgroundWorkflowDispatcher(ICommandSender commandSender, INotific
|
|||
{
|
||||
// Emit dispatching notification
|
||||
await notificationSender.SendAsync(new WorkflowDefinitionDispatching(request), cancellationToken);
|
||||
|
||||
var command = new DispatchWorkflowDefinitionCommand(request.DefinitionVersionId)
|
||||
{
|
||||
Input = request.Input,
|
||||
Properties = request.Properties,
|
||||
CorrelationId = request.CorrelationId,
|
||||
InstanceId = request.InstanceId,
|
||||
TriggerActivityId = request.TriggerActivityId,
|
||||
ParentWorkflowInstanceId = request.ParentWorkflowInstanceId,
|
||||
SchedulingActivityExecutionId = request.SchedulingActivityExecutionId,
|
||||
SchedulingWorkflowInstanceId = request.SchedulingWorkflowInstanceId,
|
||||
SchedulingCallStackDepth = request.SchedulingCallStackDepth
|
||||
};
|
||||
var command = WorkflowDispatchCommandFactory.CreateCommand(request);
|
||||
|
||||
// Background commands run independently of caller's lifecycle.
|
||||
await commandSender.SendAsync(command, CommandStrategy.Background, CreateHeaders(), CancellationToken.None);
|
||||
|
|
@ -48,13 +36,7 @@ public class BackgroundWorkflowDispatcher(ICommandSender commandSender, INotific
|
|||
{
|
||||
// Emit dispatching notification
|
||||
await notificationSender.SendAsync(new WorkflowInstanceDispatching(request), cancellationToken);
|
||||
|
||||
var command = new DispatchWorkflowInstanceCommand(request.InstanceId){
|
||||
BookmarkId = request.BookmarkId,
|
||||
ActivityHandle = request.ActivityHandle,
|
||||
Input = request.Input,
|
||||
Properties = request.Properties,
|
||||
CorrelationId = request.CorrelationId};
|
||||
var command = WorkflowDispatchCommandFactory.CreateCommand(request);
|
||||
|
||||
// Background commands run independently of caller's lifecycle.
|
||||
await commandSender.SendAsync(command, CommandStrategy.Background, CreateHeaders(), CancellationToken.None);
|
||||
|
|
@ -69,14 +51,7 @@ public class BackgroundWorkflowDispatcher(ICommandSender commandSender, INotific
|
|||
/// <inheritdoc />
|
||||
public async Task<DispatchWorkflowResponse> DispatchAsync(DispatchTriggerWorkflowsRequest request, DispatchWorkflowOptions? options = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var command = new DispatchTriggerWorkflowsCommand(request.ActivityTypeName, request.BookmarkPayload)
|
||||
{
|
||||
CorrelationId = request.CorrelationId,
|
||||
WorkflowInstanceId = request.WorkflowInstanceId,
|
||||
ActivityInstanceId = request.ActivityInstanceId,
|
||||
Input = request.Input,
|
||||
Properties = request.Properties
|
||||
};
|
||||
var command = WorkflowDispatchCommandFactory.CreateCommand(request);
|
||||
|
||||
// Background commands run independently of caller's lifecycle.
|
||||
await commandSender.SendAsync(command, CommandStrategy.Background, CreateHeaders(), CancellationToken.None);
|
||||
|
|
@ -86,13 +61,7 @@ public class BackgroundWorkflowDispatcher(ICommandSender commandSender, INotific
|
|||
/// <inheritdoc />
|
||||
public async Task<DispatchWorkflowResponse> DispatchAsync(DispatchResumeWorkflowsRequest request, DispatchWorkflowOptions? options = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var command = new DispatchResumeWorkflowsCommand(request.ActivityTypeName, request.BookmarkPayload)
|
||||
{
|
||||
CorrelationId = request.CorrelationId,
|
||||
WorkflowInstanceId = request.WorkflowInstanceId,
|
||||
ActivityInstanceId = request.ActivityInstanceId,
|
||||
Input = request.Input
|
||||
};
|
||||
var command = WorkflowDispatchCommandFactory.CreateCommand(request);
|
||||
|
||||
// Background commands run independently of caller's lifecycle.
|
||||
await commandSender.SendAsync(command, CommandStrategy.Background, CreateHeaders(), CancellationToken.None);
|
||||
|
|
@ -103,4 +72,4 @@ public class BackgroundWorkflowDispatcher(ICommandSender commandSender, INotific
|
|||
{
|
||||
return TenantHeaders.CreateHeaders(tenantAccessor.Tenant?.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,358 @@
|
|||
using Elsa.KeyValues.Contracts;
|
||||
using Elsa.KeyValues.Entities;
|
||||
using Elsa.KeyValues.Models;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Stores workflow dispatch outbox items in the existing key-value store.
|
||||
/// </summary>
|
||||
public class KeyValueWorkflowDispatchOutboxStore(IKeyValueStore keyValueStore, IPayloadSerializer payloadSerializer) : IWorkflowDispatchOutboxStore
|
||||
{
|
||||
private const string LegacyKeyPrefix = "Elsa:WorkflowDispatchOutbox:";
|
||||
private const string ItemKeyPrefix = "Elsa:WorkflowDispatchOutbox:Items:";
|
||||
private const string IndexKeyPrefix = "Elsa:WorkflowDispatchOutbox:Index:";
|
||||
private const string IndexByIdKeyPrefix = "Elsa:WorkflowDispatchOutbox:IndexById:";
|
||||
private const string RecoveryKeyPrefix = "Elsa:WorkflowDispatchOutbox:Recovery:";
|
||||
private const string StateKeyPrefix = "Elsa:WorkflowDispatchOutbox:State:";
|
||||
private const string LegacyScanCompletedKey = $"{StateKeyPrefix}LegacyScanCompleted";
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task SaveAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await keyValueStore.SaveAsync(new SerializedKeyValuePair
|
||||
{
|
||||
Key = GetRecoveryKey(item.Id),
|
||||
SerializedValue = item.Id,
|
||||
TenantId = item.TenantId
|
||||
}, cancellationToken);
|
||||
|
||||
await keyValueStore.SaveAsync(new SerializedKeyValuePair
|
||||
{
|
||||
Key = GetItemKey(item.Id),
|
||||
SerializedValue = payloadSerializer.Serialize(item),
|
||||
TenantId = item.TenantId
|
||||
}, cancellationToken);
|
||||
|
||||
await SaveIndexAsync(item, cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetRecoveryKey(item.Id), cancellationToken);
|
||||
}
|
||||
|
||||
private async Task SaveIndexAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var indexKey = GetIndexKey(item);
|
||||
|
||||
await keyValueStore.SaveAsync(new SerializedKeyValuePair
|
||||
{
|
||||
Key = indexKey,
|
||||
SerializedValue = item.Id,
|
||||
TenantId = item.TenantId
|
||||
}, cancellationToken);
|
||||
|
||||
await keyValueStore.SaveAsync(new SerializedKeyValuePair
|
||||
{
|
||||
Key = GetIndexByIdKey(item.Id),
|
||||
SerializedValue = indexKey,
|
||||
TenantId = item.TenantId
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<WorkflowDispatchOutboxItem>> FindManyAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await FindManyAsync(maxCount: 0, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<WorkflowDispatchOutboxItem>> FindManyAsync(int maxCount, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var indexedItems = await FindIndexedItemsAsync(maxCount, cancellationToken);
|
||||
var recoverableItems = await FindRecoveryItemsAsync(maxCount, cancellationToken);
|
||||
var legacyItems = await FindLegacyItemsAsync(maxCount, cancellationToken);
|
||||
var items = indexedItems
|
||||
.Concat(recoverableItems)
|
||||
.Concat(legacyItems)
|
||||
.GroupBy(x => x.Id)
|
||||
.Select(x => x.First())
|
||||
.OrderBy(x => x.CreatedAt);
|
||||
|
||||
if (maxCount > 0)
|
||||
return items.Take(maxCount).ToList();
|
||||
|
||||
return items.ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DeleteAsync(string id, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var itemKey = GetItemKey(id);
|
||||
var record = await keyValueStore.FindAsync(new KeyValueFilter { Key = itemKey }, cancellationToken);
|
||||
|
||||
if (record == null)
|
||||
{
|
||||
await DeleteIndexesForMissingItemAsync(id, cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetRecoveryKey(id), cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetLegacyKey(id), cancellationToken);
|
||||
await keyValueStore.DeleteAsync(itemKey, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var item = payloadSerializer.Deserialize<WorkflowDispatchOutboxItem>(record.SerializedValue);
|
||||
|
||||
if (item != null)
|
||||
{
|
||||
await keyValueStore.DeleteAsync(GetIndexKey(item), cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetIndexByIdKey(id), cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await DeleteIndexesForMissingItemAsync(id, cancellationToken);
|
||||
}
|
||||
|
||||
await keyValueStore.DeleteAsync(GetRecoveryKey(id), cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetLegacyKey(id), cancellationToken);
|
||||
await keyValueStore.DeleteAsync(itemKey, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<WorkflowDispatchOutboxItem>> FindIndexedItemsAsync(int maxCount, CancellationToken cancellationToken)
|
||||
{
|
||||
var indexRecords = (await keyValueStore.FindManyAsync(new KeyValueFilter
|
||||
{
|
||||
Key = IndexKeyPrefix,
|
||||
StartsWith = true,
|
||||
OrderByKey = true,
|
||||
Take = maxCount > 0 ? maxCount : null
|
||||
}, cancellationToken)).ToList();
|
||||
var itemKeys = indexRecords.Select(x => GetItemKey(x.SerializedValue)).ToList();
|
||||
|
||||
var itemRecords = itemKeys.Count == 0
|
||||
? Array.Empty<SerializedKeyValuePair>()
|
||||
: await keyValueStore.FindManyAsync(new KeyValueFilter
|
||||
{
|
||||
Keys = itemKeys
|
||||
}, cancellationToken);
|
||||
var itemRecordLookup = itemRecords.ToDictionary(x => x.Key);
|
||||
var items = new List<WorkflowDispatchOutboxItem>();
|
||||
|
||||
foreach (var indexRecord in indexRecords)
|
||||
{
|
||||
var id = indexRecord.SerializedValue;
|
||||
var itemKey = GetItemKey(id);
|
||||
|
||||
if (!itemRecordLookup.TryGetValue(itemKey, out var itemRecord))
|
||||
{
|
||||
await keyValueStore.DeleteAsync(indexRecord.Key, cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetIndexByIdKey(id), cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
var item = payloadSerializer.Deserialize<WorkflowDispatchOutboxItem>(itemRecord.SerializedValue);
|
||||
|
||||
if (item == null)
|
||||
{
|
||||
await DeleteUnrecoverableItemAsync(id, itemKey, indexRecord.Key, recoveryKey: null, cancellationToken: cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<WorkflowDispatchOutboxItem>> FindRecoveryItemsAsync(int maxCount, CancellationToken cancellationToken)
|
||||
{
|
||||
var recoveryRecords = (await keyValueStore.FindManyAsync(new KeyValueFilter
|
||||
{
|
||||
Key = RecoveryKeyPrefix,
|
||||
StartsWith = true,
|
||||
OrderByKey = true,
|
||||
Take = maxCount > 0 ? maxCount : null
|
||||
}, cancellationToken)).ToList();
|
||||
|
||||
if (recoveryRecords.Count == 0)
|
||||
return [];
|
||||
|
||||
var itemKeys = recoveryRecords.Select(x => GetItemKey(x.SerializedValue)).ToList();
|
||||
var itemRecords = (await keyValueStore.FindManyAsync(new KeyValueFilter { Keys = itemKeys }, cancellationToken)).ToDictionary(x => x.Key);
|
||||
var items = new List<WorkflowDispatchOutboxItem>();
|
||||
|
||||
foreach (var recoveryRecord in recoveryRecords)
|
||||
{
|
||||
var id = recoveryRecord.SerializedValue;
|
||||
|
||||
if (!itemRecords.TryGetValue(GetItemKey(id), out var itemRecord))
|
||||
{
|
||||
await DeleteIndexesForMissingItemAsync(id, cancellationToken);
|
||||
await keyValueStore.DeleteAsync(recoveryRecord.Key, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
var item = payloadSerializer.Deserialize<WorkflowDispatchOutboxItem>(itemRecord.SerializedValue);
|
||||
|
||||
if (item == null)
|
||||
{
|
||||
await DeleteUnrecoverableItemAsync(id, itemRecord.Key, indexKey: null, recoveryKey: recoveryRecord.Key, cancellationToken: cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
await SaveIndexAsync(item, cancellationToken);
|
||||
await keyValueStore.DeleteAsync(recoveryRecord.Key, cancellationToken);
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<WorkflowDispatchOutboxItem>> FindLegacyItemsAsync(int maxCount, CancellationToken cancellationToken)
|
||||
{
|
||||
var legacyScanCompleted = await keyValueStore.FindAsync(new KeyValueFilter { Key = LegacyScanCompletedKey }, cancellationToken);
|
||||
|
||||
if (legacyScanCompleted != null)
|
||||
return [];
|
||||
|
||||
var records = await keyValueStore.FindManyAsync(new KeyValueFilter
|
||||
{
|
||||
Key = LegacyKeyPrefix,
|
||||
StartsWith = true,
|
||||
OrderByKey = true,
|
||||
Take = GetLegacyScanTake(maxCount)
|
||||
}, cancellationToken);
|
||||
var recordList = records.ToList();
|
||||
|
||||
var recoverableItems = recordList
|
||||
.Where(IsRecoverableItemRecord)
|
||||
.Select(x => new
|
||||
{
|
||||
Record = x,
|
||||
Item = payloadSerializer.Deserialize<WorkflowDispatchOutboxItem>(x.SerializedValue)
|
||||
})
|
||||
.Where(x => x.Item != null)
|
||||
.ToList();
|
||||
var recoverableItemsToMigrate = maxCount > 0 ? recoverableItems.Take(maxCount).ToList() : recoverableItems;
|
||||
var items = new List<WorkflowDispatchOutboxItem>();
|
||||
|
||||
foreach (var recoverableItem in recoverableItemsToMigrate)
|
||||
{
|
||||
var record = recoverableItem.Record;
|
||||
var item = recoverableItem.Item!;
|
||||
await SaveAsync(item, cancellationToken);
|
||||
|
||||
if (!record.Key.StartsWith(ItemKeyPrefix, StringComparison.Ordinal))
|
||||
await keyValueStore.DeleteAsync(record.Key, cancellationToken);
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
if (IsLegacyScanComplete(recordList.Count, maxCount))
|
||||
{
|
||||
await keyValueStore.SaveAsync(new SerializedKeyValuePair
|
||||
{
|
||||
Key = LegacyScanCompletedKey,
|
||||
SerializedValue = "true"
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private async Task DeleteIndexesForMissingItemAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
var lookupRecord = await keyValueStore.FindAsync(new KeyValueFilter { Key = GetIndexByIdKey(id) }, cancellationToken);
|
||||
|
||||
if (lookupRecord != null)
|
||||
{
|
||||
if (IsIndexKeyForId(lookupRecord.SerializedValue, id))
|
||||
{
|
||||
await keyValueStore.DeleteAsync(lookupRecord.SerializedValue, cancellationToken);
|
||||
await keyValueStore.DeleteAsync(lookupRecord.Key, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
await keyValueStore.DeleteAsync(lookupRecord.Key, cancellationToken);
|
||||
}
|
||||
|
||||
var matchingIndexRecords = await keyValueStore.FindManyAsync(new KeyValueFilter
|
||||
{
|
||||
Key = IndexKeyPrefix,
|
||||
StartsWith = true
|
||||
}, cancellationToken);
|
||||
|
||||
foreach (var indexRecord in matchingIndexRecords.Where(x => x.SerializedValue == id || x.Key.EndsWith($":{id}", StringComparison.Ordinal)))
|
||||
{
|
||||
await keyValueStore.DeleteAsync(indexRecord.Key, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DeleteUnrecoverableItemAsync(string id, string itemKey, string? indexKey, string? recoveryKey, CancellationToken cancellationToken)
|
||||
{
|
||||
if (indexKey != null)
|
||||
{
|
||||
await keyValueStore.DeleteAsync(indexKey, cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetIndexByIdKey(id), cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await DeleteIndexesForMissingItemAsync(id, cancellationToken);
|
||||
}
|
||||
|
||||
await keyValueStore.DeleteAsync(recoveryKey ?? GetRecoveryKey(id), cancellationToken);
|
||||
await keyValueStore.DeleteAsync(GetLegacyKey(id), cancellationToken);
|
||||
await keyValueStore.DeleteAsync(itemKey, cancellationToken);
|
||||
}
|
||||
|
||||
private static string GetItemKey(string id) => $"{ItemKeyPrefix}{id}";
|
||||
|
||||
private static string GetLegacyKey(string id) => $"{LegacyKeyPrefix}{id}";
|
||||
|
||||
private static string GetIndexKey(WorkflowDispatchOutboxItem item) => $"{IndexKeyPrefix}{item.CreatedAt.UtcTicks:D20}:{item.Id}";
|
||||
|
||||
private static string GetIndexByIdKey(string id) => $"{IndexByIdKeyPrefix}{id}";
|
||||
|
||||
private static string GetRecoveryKey(string id) => $"{RecoveryKeyPrefix}{id}";
|
||||
|
||||
private static int? GetLegacyScanTake(int maxCount)
|
||||
{
|
||||
if (maxCount <= 0)
|
||||
return null;
|
||||
|
||||
return maxCount == int.MaxValue ? int.MaxValue : maxCount + 1;
|
||||
}
|
||||
|
||||
private static bool IsLegacyScanComplete(int recordCount, int maxCount)
|
||||
{
|
||||
var scanTake = GetLegacyScanTake(maxCount);
|
||||
return scanTake == null || recordCount < scanTake.Value;
|
||||
}
|
||||
|
||||
private static bool IsIndexKeyForId(string indexKey, string id)
|
||||
{
|
||||
var expectedSuffix = $":{id}";
|
||||
|
||||
if (!indexKey.StartsWith(IndexKeyPrefix, StringComparison.Ordinal) || !indexKey.EndsWith(expectedSuffix, StringComparison.Ordinal))
|
||||
return false;
|
||||
|
||||
var ticksStart = IndexKeyPrefix.Length;
|
||||
var ticksLength = indexKey.Length - ticksStart - expectedSuffix.Length;
|
||||
|
||||
if (ticksLength != 20)
|
||||
return false;
|
||||
|
||||
foreach (var character in indexKey.AsSpan(ticksStart, ticksLength))
|
||||
{
|
||||
if (character is < '0' or > '9')
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool IsRecoverableItemRecord(SerializedKeyValuePair record)
|
||||
{
|
||||
return !record.Key.StartsWith(IndexKeyPrefix, StringComparison.Ordinal)
|
||||
&& !record.Key.StartsWith(IndexByIdKeyPrefix, StringComparison.Ordinal)
|
||||
&& !record.Key.StartsWith(RecoveryKeyPrefix, StringComparison.Ordinal)
|
||||
&& !record.Key.StartsWith(StateKeyPrefix, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
using System.Diagnostics.CodeAnalysis;
|
||||
using Elsa.Common;
|
||||
using Elsa.Common.Multitenancy;
|
||||
using Elsa.Mediator.Contracts;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
using Elsa.Workflows.Runtime.Notifications;
|
||||
using Elsa.Workflows.Runtime.Options;
|
||||
using Elsa.Workflows.Runtime.Requests;
|
||||
using Elsa.Workflows.Runtime.Responses;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// Writes workflow dispatch commands to a durable outbox while a workflow execution is in progress.
|
||||
/// </summary>
|
||||
public class TransactionalWorkflowDispatcher(
|
||||
IWorkflowDispatcher decoratedService,
|
||||
IServiceProvider serviceProvider,
|
||||
IWorkflowDispatchOutboxAccessor outboxAccessor,
|
||||
INotificationSender notificationSender,
|
||||
IIdentityGenerator identityGenerator,
|
||||
IOptions<WorkflowDispatcherOptions> options,
|
||||
ITenantAccessor? tenantAccessor = null) : IWorkflowDispatcher
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async Task<DispatchWorkflowResponse> DispatchAsync(DispatchWorkflowDefinitionRequest request, DispatchWorkflowOptions? dispatchOptions = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!TryGetWorkflowExecutionContext(out var context))
|
||||
return await decoratedService.DispatchAsync(request, dispatchOptions, cancellationToken);
|
||||
|
||||
await notificationSender.SendAsync(new WorkflowDefinitionDispatching(request), cancellationToken);
|
||||
var generatedInstanceId = string.IsNullOrWhiteSpace(request.InstanceId) ? identityGenerator.GenerateId() : null;
|
||||
|
||||
await GetOutbox().EnqueueAsync(context, new()
|
||||
{
|
||||
TenantId = tenantAccessor?.Tenant?.Id,
|
||||
Kind = WorkflowDispatchOutboxItemKind.WorkflowDefinition,
|
||||
WorkflowDefinitionCommand = WorkflowDispatchCommandFactory.CreateCommand(request, generatedInstanceId)
|
||||
}, cancellationToken);
|
||||
|
||||
var response = DispatchWorkflowResponse.Success();
|
||||
await notificationSender.SendAsync(new WorkflowDefinitionDispatched(request, response), cancellationToken);
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DispatchWorkflowResponse> DispatchAsync(DispatchWorkflowInstanceRequest request, DispatchWorkflowOptions? dispatchOptions = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!TryGetWorkflowExecutionContext(out var context))
|
||||
return await decoratedService.DispatchAsync(request, dispatchOptions, cancellationToken);
|
||||
|
||||
await notificationSender.SendAsync(new WorkflowInstanceDispatching(request), cancellationToken);
|
||||
|
||||
await GetOutbox().EnqueueAsync(context, new()
|
||||
{
|
||||
TenantId = tenantAccessor?.Tenant?.Id,
|
||||
Kind = WorkflowDispatchOutboxItemKind.WorkflowInstance,
|
||||
WorkflowInstanceCommand = WorkflowDispatchCommandFactory.CreateCommand(request)
|
||||
}, cancellationToken);
|
||||
|
||||
var response = DispatchWorkflowResponse.Success();
|
||||
await notificationSender.SendAsync(new WorkflowInstanceDispatched(request, response), cancellationToken);
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DispatchWorkflowResponse> DispatchAsync(DispatchTriggerWorkflowsRequest request, DispatchWorkflowOptions? dispatchOptions = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!TryGetWorkflowExecutionContext(out var context))
|
||||
return await decoratedService.DispatchAsync(request, dispatchOptions, cancellationToken);
|
||||
|
||||
await GetOutbox().EnqueueAsync(context, new()
|
||||
{
|
||||
TenantId = tenantAccessor?.Tenant?.Id,
|
||||
Kind = WorkflowDispatchOutboxItemKind.TriggerWorkflows,
|
||||
TriggerWorkflowsCommand = WorkflowDispatchCommandFactory.CreateCommand(request)
|
||||
}, cancellationToken);
|
||||
|
||||
return DispatchWorkflowResponse.Success();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DispatchWorkflowResponse> DispatchAsync(DispatchResumeWorkflowsRequest request, DispatchWorkflowOptions? dispatchOptions = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!TryGetWorkflowExecutionContext(out var context))
|
||||
return await decoratedService.DispatchAsync(request, dispatchOptions, cancellationToken);
|
||||
|
||||
await GetOutbox().EnqueueAsync(context, new()
|
||||
{
|
||||
TenantId = tenantAccessor?.Tenant?.Id,
|
||||
Kind = WorkflowDispatchOutboxItemKind.ResumeWorkflows,
|
||||
ResumeWorkflowsCommand = WorkflowDispatchCommandFactory.CreateCommand(request)
|
||||
}, cancellationToken);
|
||||
|
||||
return DispatchWorkflowResponse.Success();
|
||||
}
|
||||
|
||||
private bool TryGetWorkflowExecutionContext([NotNullWhen(true)] out WorkflowExecutionContext? context)
|
||||
{
|
||||
var currentContext = outboxAccessor.WorkflowExecutionContext;
|
||||
if (!options.Value.UseTransactionalOutbox || currentContext == null)
|
||||
{
|
||||
context = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
context = currentContext;
|
||||
return true;
|
||||
}
|
||||
|
||||
private IWorkflowDispatchOutbox GetOutbox() => serviceProvider.GetRequiredService<IWorkflowDispatchOutbox>();
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
using Elsa.Common;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <inheritdoc />
|
||||
public class WorkflowDispatchOutbox(
|
||||
IWorkflowDispatchOutboxStore store,
|
||||
IIdentityGenerator identityGenerator,
|
||||
ISystemClock systemClock) : IWorkflowDispatchOutbox
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async Task EnqueueAsync(WorkflowExecutionContext workflowExecutionContext, WorkflowDispatchOutboxItem item, CancellationToken cancellationToken = default)
|
||||
{
|
||||
item.Id = identityGenerator.GenerateId();
|
||||
item.OwnerWorkflowInstanceId = workflowExecutionContext.Id;
|
||||
item.CreatedAt = systemClock.UtcNow;
|
||||
|
||||
await store.SaveAsync(item, cancellationToken);
|
||||
workflowExecutionContext.AddWorkflowDispatchOutboxItem(item.Id);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <inheritdoc />
|
||||
public class WorkflowDispatchOutboxAccessor : IWorkflowDispatchOutboxAccessor
|
||||
{
|
||||
private readonly AsyncLocal<WorkflowExecutionContext?> _currentWorkflowExecutionContext = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public WorkflowExecutionContext? WorkflowExecutionContext
|
||||
{
|
||||
get => _currentWorkflowExecutionContext.Value;
|
||||
set => _currentWorkflowExecutionContext.Value = value;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,324 @@
|
|||
using Elsa.Common;
|
||||
using Elsa.Common.DistributedHosting;
|
||||
using Elsa.Common.Models;
|
||||
using Elsa.Common.Multitenancy;
|
||||
using Elsa.Mediator;
|
||||
using Elsa.Mediator.Contracts;
|
||||
using Elsa.Tenants.Mediator;
|
||||
using Elsa.Workflows.Management;
|
||||
using Elsa.Workflows.Management.Entities;
|
||||
using Elsa.Workflows.Management.Filters;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
using Elsa.Workflows.Runtime.Options;
|
||||
using Medallion.Threading;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Elsa.Workflows.Runtime;
|
||||
|
||||
/// <inheritdoc />
|
||||
public class WorkflowDispatchOutboxProcessor(
|
||||
IWorkflowDispatchOutboxStore store,
|
||||
IWorkflowInstanceStore workflowInstanceStore,
|
||||
ICommandSender commandSender,
|
||||
IDistributedLockProvider distributedLockProvider,
|
||||
ISystemClock systemClock,
|
||||
IOptions<DistributedLockingOptions> distributedLockingOptions,
|
||||
IOptions<WorkflowDispatcherOptions> dispatcherOptions,
|
||||
ILogger<WorkflowDispatchOutboxProcessor> logger,
|
||||
ITenantAccessor? tenantAccessor = null) : IWorkflowDispatchOutboxProcessor
|
||||
{
|
||||
private const string LockResource = "Elsa:WorkflowDispatchOutbox:Processor";
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ProcessAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
IDistributedSynchronizationHandle? handle;
|
||||
|
||||
try
|
||||
{
|
||||
handle = await distributedLockProvider.TryAcquireLockAsync(GetLockResource(), distributedLockingOptions.Value.LockAcquisitionTimeout, cancellationToken);
|
||||
}
|
||||
catch (TimeoutException e) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogDebug(e, "Skipping workflow dispatch outbox processing because the processor lock could not be acquired.");
|
||||
return;
|
||||
}
|
||||
|
||||
await using (handle)
|
||||
{
|
||||
if (handle == null)
|
||||
{
|
||||
logger.LogDebug("Skipping workflow dispatch outbox processing because another processor owns the lock.");
|
||||
return;
|
||||
}
|
||||
|
||||
await ProcessPendingItemsAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TryProcessAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
IDistributedSynchronizationHandle? handle;
|
||||
|
||||
try
|
||||
{
|
||||
handle = await distributedLockProvider.TryAcquireLockAsync(GetLockResource(), TimeSpan.Zero, cancellationToken);
|
||||
}
|
||||
catch (TimeoutException e) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogDebug(e, "Skipping eager workflow dispatch outbox processing because the processor lock could not be acquired.");
|
||||
return false;
|
||||
}
|
||||
|
||||
await using (handle)
|
||||
{
|
||||
if (handle == null)
|
||||
{
|
||||
logger.LogDebug("Skipping eager workflow dispatch outbox processing because another processor owns the lock.");
|
||||
return false;
|
||||
}
|
||||
|
||||
await ProcessPendingItemsAsync(cancellationToken);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessPendingItemsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var batchSize = Math.Max(1, dispatcherOptions.Value.OutboxProcessorBatchSize);
|
||||
var items = (await store.FindManyAsync(batchSize, cancellationToken)).ToList();
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessAsync(item, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(e, "Failed to process workflow dispatch outbox item {OutboxItemId}; processing will continue with the next item.", item.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string GetLockResource()
|
||||
{
|
||||
var tenantId = tenantAccessor?.Tenant?.Id;
|
||||
return string.IsNullOrWhiteSpace(tenantId) ? LockResource : $"{LockResource}:{tenantId}";
|
||||
}
|
||||
|
||||
private async Task ProcessAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var owner = await workflowInstanceStore.FindAsync(new WorkflowInstanceFilter { Id = item.OwnerWorkflowInstanceId }, cancellationToken);
|
||||
|
||||
if (owner == null)
|
||||
{
|
||||
await HandleMissingOwnerAsync(item, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!owner.WorkflowState.HasWorkflowDispatchOutboxItem(item.Id))
|
||||
{
|
||||
await HandleUncommittedOwnerAsync(item, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await SendAsync(item, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
await HandleDeliveryFailureAsync(item, e, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
await CleanupDeliveredItemAsync(item, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task SendAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var headers = TenantHeaders.CreateHeaders(item.TenantId);
|
||||
|
||||
switch (item.Kind)
|
||||
{
|
||||
case WorkflowDispatchOutboxItemKind.WorkflowDefinition when item.WorkflowDefinitionCommand != null:
|
||||
await commandSender.SendAsync(item.WorkflowDefinitionCommand, CommandStrategy.Background, headers, cancellationToken);
|
||||
break;
|
||||
case WorkflowDispatchOutboxItemKind.WorkflowInstance when item.WorkflowInstanceCommand != null:
|
||||
await commandSender.SendAsync(item.WorkflowInstanceCommand, CommandStrategy.Background, headers, cancellationToken);
|
||||
break;
|
||||
case WorkflowDispatchOutboxItemKind.TriggerWorkflows when item.TriggerWorkflowsCommand != null:
|
||||
await commandSender.SendAsync(item.TriggerWorkflowsCommand, CommandStrategy.Background, headers, cancellationToken);
|
||||
break;
|
||||
case WorkflowDispatchOutboxItemKind.ResumeWorkflows when item.ResumeWorkflowsCommand != null:
|
||||
await commandSender.SendAsync(item.ResumeWorkflowsCommand, CommandStrategy.Background, headers, cancellationToken);
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException($"Outbox item {item.Id} does not contain a dispatch command for kind {item.Kind}.");
|
||||
}
|
||||
|
||||
logger.LogDebug("Delivered workflow dispatch outbox item {OutboxItemId} for owner workflow {WorkflowInstanceId}", item.Id, item.OwnerWorkflowInstanceId);
|
||||
}
|
||||
|
||||
private async Task HandleMissingOwnerAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var retention = dispatcherOptions.Value.OrphanedOutboxItemRetention;
|
||||
var expiresAt = item.CreatedAt.Add(retention);
|
||||
|
||||
if (retention <= TimeSpan.Zero || systemClock.UtcNow >= expiresAt)
|
||||
{
|
||||
logger.LogWarning("Deleting workflow dispatch outbox item {OutboxItemId} because owner workflow {WorkflowInstanceId} was not found and the orphan retention period has elapsed", item.Id, item.OwnerWorkflowInstanceId);
|
||||
await TryDeleteRetainedItemAsync(item, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogDebug("Skipping workflow dispatch outbox item {OutboxItemId} because owner workflow {WorkflowInstanceId} was not found; it will be retained until {ExpiresAt}", item.Id, item.OwnerWorkflowInstanceId, expiresAt);
|
||||
}
|
||||
|
||||
private async Task HandleUncommittedOwnerAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var retention = dispatcherOptions.Value.OrphanedOutboxItemRetention;
|
||||
var expiresAt = item.CreatedAt.Add(retention);
|
||||
|
||||
if (retention <= TimeSpan.Zero || systemClock.UtcNow >= expiresAt)
|
||||
{
|
||||
logger.LogWarning("Deleting workflow dispatch outbox item {OutboxItemId} because owner workflow {WorkflowInstanceId} never committed the outbox marker and the retention period has elapsed", item.Id, item.OwnerWorkflowInstanceId);
|
||||
await TryDeleteRetainedItemAsync(item, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogDebug("Skipping workflow dispatch outbox item {OutboxItemId} because owner workflow {WorkflowInstanceId} has not committed it; it will be retained until {ExpiresAt}", item.Id, item.OwnerWorkflowInstanceId, expiresAt);
|
||||
}
|
||||
|
||||
private async Task TryDeleteRetainedItemAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await store.DeleteAsync(item.Id, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(e, "Failed to delete retained workflow dispatch outbox item {OutboxItemId}; it will be retried without counting as a delivery failure.", item.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleDeliveryFailureAsync(WorkflowDispatchOutboxItem item, Exception exception, CancellationToken cancellationToken)
|
||||
{
|
||||
item.DeliveryAttempts++;
|
||||
|
||||
try
|
||||
{
|
||||
if (item.DeliveryAttempts >= dispatcherOptions.Value.MaxOutboxDeliveryAttempts)
|
||||
{
|
||||
logger.LogWarning(exception, "Abandoning workflow dispatch outbox item {OutboxItemId} after {DeliveryAttempts} failed delivery attempts", item.Id, item.DeliveryAttempts);
|
||||
if (await TryDeleteAbandonedItemAsync(item, cancellationToken))
|
||||
await RemoveCommittedMarkerAsync(item, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
await store.SaveAsync(item, cancellationToken);
|
||||
logger.LogError(exception, "Failed to deliver workflow dispatch outbox item {OutboxItemId}; attempt {DeliveryAttempts} of {MaxDeliveryAttempts}", item.Id, item.DeliveryAttempts, dispatcherOptions.Value.MaxOutboxDeliveryAttempts);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(e, "Failed to persist delivery failure state for workflow dispatch outbox item {OutboxItemId}; processing will continue with the next item.", item.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> TryDeleteAbandonedItemAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await store.DeleteAsync(item.Id, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(e, "Failed to delete abandoned workflow dispatch outbox item {OutboxItemId}; preserving its delivery attempt count for the next processor run.", item.Id);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await store.SaveAsync(item, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(e, "Failed to preserve delivery attempt count for abandoned workflow dispatch outbox item {OutboxItemId}.", item.Id);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task CleanupDeliveredItemAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await store.DeleteAsync(item.Id, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(e, "Delivered workflow dispatch outbox item {OutboxItemId}, but failed to delete it from the outbox store. The item was not counted as a delivery failure.", item.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
await RemoveCommittedMarkerAsync(item, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task RemoveCommittedMarkerAsync(WorkflowDispatchOutboxItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var lockResource = $"workflow-instance:{item.OwnerWorkflowInstanceId}";
|
||||
|
||||
try
|
||||
{
|
||||
await using var handle = await distributedLockProvider.AcquireLockAsync(lockResource, distributedLockingOptions.Value.LockAcquisitionTimeout, cancellationToken);
|
||||
var owner = await workflowInstanceStore.FindAsync(new WorkflowInstanceFilter { Id = item.OwnerWorkflowInstanceId }, cancellationToken);
|
||||
|
||||
if (owner == null)
|
||||
return;
|
||||
|
||||
if (!owner.WorkflowState.RemoveWorkflowDispatchOutboxItem(item.Id))
|
||||
return;
|
||||
|
||||
await workflowInstanceStore.SaveAsync(owner, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e) when (e is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(e, "Failed to remove delivered workflow dispatch outbox item {OutboxItemId} from owner workflow {WorkflowInstanceId}; future commits may prune the marker during a later sweep.", item.Id, item.OwnerWorkflowInstanceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ using Elsa.Workflows.Runtime.UIHints;
|
|||
using Medallion.Threading;
|
||||
using Medallion.Threading.FileSystem;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.ShellFeatures;
|
||||
|
|
@ -50,7 +51,8 @@ public class WorkflowRuntimeFeature : IShellFeature
|
|||
public Func<IServiceProvider, IWorkflowDispatcher> WorkflowDispatcher { get; set; } = sp =>
|
||||
{
|
||||
var decoratedService = ActivatorUtilities.CreateInstance<BackgroundWorkflowDispatcher>(sp);
|
||||
return ActivatorUtilities.CreateInstance<ValidatingWorkflowDispatcher>(sp, decoratedService);
|
||||
var transactionalService = ActivatorUtilities.CreateInstance<TransactionalWorkflowDispatcher>(sp, decoratedService);
|
||||
return ActivatorUtilities.CreateInstance<ValidatingWorkflowDispatcher>(sp, transactionalService);
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -63,6 +65,11 @@ public class WorkflowRuntimeFeature : IShellFeature
|
|||
/// </summary>
|
||||
public Func<IServiceProvider, IWorkflowCancellationDispatcher> WorkflowCancellationDispatcher { get; set; } = sp => ActivatorUtilities.CreateInstance<BackgroundWorkflowCancellationDispatcher>(sp);
|
||||
|
||||
/// <summary>
|
||||
/// A factory that instantiates an <see cref="IWorkflowDispatchOutboxStore"/>.
|
||||
/// </summary>
|
||||
public Func<IServiceProvider, IWorkflowDispatchOutboxStore> WorkflowDispatchOutboxStore { get; set; } = sp => ActivatorUtilities.CreateInstance<KeyValueWorkflowDispatchOutboxStore>(sp);
|
||||
|
||||
/// <summary>
|
||||
/// A factory that instantiates an <see cref="IBookmarkStore"/>.
|
||||
/// </summary>
|
||||
|
|
@ -235,6 +242,7 @@ public class WorkflowRuntimeFeature : IShellFeature
|
|||
.AddScoped<IWorkflowStarter, DefaultWorkflowStarter>()
|
||||
.AddScoped<IWorkflowRestarter, DefaultWorkflowRestarter>()
|
||||
.AddScoped<IBookmarkQueuePurger, DefaultBookmarkQueuePurger>()
|
||||
.AddSingleton<IWorkflowDispatchOutboxAccessor, WorkflowDispatchOutboxAccessor>()
|
||||
.AddScoped<ILogRecordExtractor<WorkflowExecutionLogRecord>, WorkflowExecutionLogRecordExtractor>()
|
||||
.AddScoped<IActivityPropertyLogPersistenceEvaluator, ActivityPropertyLogPersistenceEvaluator>()
|
||||
.AddScoped<IBookmarkQueueProcessor, BookmarkQueueProcessor>()
|
||||
|
|
@ -276,6 +284,7 @@ public class WorkflowRuntimeFeature : IShellFeature
|
|||
.AddRecurringTask<TriggerBookmarkQueueRecurringTask>(TimeSpan.FromMinutes(1))
|
||||
.AddRecurringTask<PurgeBookmarkQueueRecurringTask>(TimeSpan.FromSeconds(10))
|
||||
.AddRecurringTask<RestartInterruptedWorkflowsTask>(TimeSpan.FromMinutes(5)) // Same default as the workflow liveness threshold.
|
||||
.AddRecurringTask<ProcessWorkflowDispatchOutboxRecurringTask>(TimeSpan.FromSeconds(10))
|
||||
|
||||
// Distributed locking.
|
||||
.AddSingleton(DistributedLockProvider)
|
||||
|
|
@ -292,6 +301,7 @@ public class WorkflowRuntimeFeature : IShellFeature
|
|||
.AddCommandHandler<CancelWorkflowsCommandHandler>()
|
||||
.AddNotificationHandler<ResumeDispatchWorkflowActivity>()
|
||||
.AddNotificationHandler<ResumeBulkDispatchWorkflowActivity>()
|
||||
.AddNotificationHandler<ProcessWorkflowDispatchOutbox>()
|
||||
.AddNotificationHandler<ResumeExecuteWorkflowActivity>()
|
||||
.AddNotificationHandler<IndexTriggers>()
|
||||
.AddNotificationHandler<CancelBackgroundActivities>()
|
||||
|
|
@ -310,5 +320,9 @@ public class WorkflowRuntimeFeature : IShellFeature
|
|||
.AddScoped<IWorkflowActivationStrategy, CorrelatedSingletonStrategy>()
|
||||
.AddScoped<IWorkflowActivationStrategy, CorrelationStrategy>()
|
||||
;
|
||||
|
||||
services.TryAddScoped<IWorkflowDispatchOutbox>(sp => ActivatorUtilities.CreateInstance<WorkflowDispatchOutbox>(sp));
|
||||
services.TryAddScoped(WorkflowDispatchOutboxStore);
|
||||
services.TryAddScoped<IWorkflowDispatchOutboxProcessor, WorkflowDispatchOutboxProcessor>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
using Elsa.Common;
|
||||
using Elsa.Common.RecurringTasks;
|
||||
using Elsa.Workflows.Runtime.Options;
|
||||
using JetBrains.Annotations;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// Periodically processes pending workflow dispatch outbox items.
|
||||
/// </summary>
|
||||
[SingleNodeTask]
|
||||
[UsedImplicitly]
|
||||
public class ProcessWorkflowDispatchOutboxRecurringTask(IServiceProvider serviceProvider, IOptions<WorkflowDispatcherOptions> options) : RecurringTask
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override async Task ExecuteAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!options.Value.UseTransactionalOutbox)
|
||||
return;
|
||||
|
||||
var processor = serviceProvider.GetRequiredService<IWorkflowDispatchOutboxProcessor>();
|
||||
await processor.ProcessAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
using Elsa.Workflows.Runtime.Requests;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.UnitTests.Extensions;
|
||||
|
||||
public class WorkflowDispatchCommandFactoryTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
public void CreateCommand_UsesFallbackInstanceId_WhenRequestDoesNotSpecifyInstanceId(string? requestInstanceId)
|
||||
{
|
||||
var request = new DispatchWorkflowDefinitionRequest("definition-version-1")
|
||||
{
|
||||
InstanceId = requestInstanceId
|
||||
};
|
||||
|
||||
var command = WorkflowDispatchCommandFactory.CreateCommand(request, "generated-instance");
|
||||
|
||||
Assert.Equal("generated-instance", command.InstanceId);
|
||||
Assert.True(command.SkipIfInstanceExists);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateCommand_PreservesRequestInstanceId_WhenSpecified()
|
||||
{
|
||||
var request = new DispatchWorkflowDefinitionRequest("definition-version-1")
|
||||
{
|
||||
InstanceId = "requested-instance"
|
||||
};
|
||||
|
||||
var command = WorkflowDispatchCommandFactory.CreateCommand(request, "generated-instance");
|
||||
|
||||
Assert.Equal("requested-instance", command.InstanceId);
|
||||
Assert.False(command.SkipIfInstanceExists);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateCommand_DoesNotSkipExistingInstance_WhenFallbackInstanceIdIsMissing()
|
||||
{
|
||||
var request = new DispatchWorkflowDefinitionRequest("definition-version-1");
|
||||
|
||||
var command = WorkflowDispatchCommandFactory.CreateCommand(request);
|
||||
|
||||
Assert.Null(command.InstanceId);
|
||||
Assert.False(command.SkipIfInstanceExists);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
using Elsa.Workflows.Runtime.Commands;
|
||||
using Elsa.Workflows.Runtime.Handlers;
|
||||
using Elsa.Workflows.Runtime.Messages;
|
||||
using NSubstitute;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.UnitTests.Handlers;
|
||||
|
||||
public class DispatchWorkflowCommandHandlerTests
|
||||
{
|
||||
private readonly IStimulusSender _stimulusSender = Substitute.For<IStimulusSender>();
|
||||
private readonly IWorkflowRuntime _workflowRuntime = Substitute.For<IWorkflowRuntime>();
|
||||
private readonly IWorkflowClient _workflowClient = Substitute.For<IWorkflowClient>();
|
||||
private readonly DispatchWorkflowCommandHandler _handler;
|
||||
|
||||
public DispatchWorkflowCommandHandlerTests()
|
||||
{
|
||||
_handler = new(_stimulusSender, _workflowRuntime);
|
||||
_workflowRuntime.CreateClientAsync("child-1", Arg.Any<CancellationToken>()).Returns(new ValueTask<IWorkflowClient>(_workflowClient));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleAsync_DoesNotCheckExistingInstanceAndCreatesAndRunsWorkflow_WhenIdempotencyIsNotRequested()
|
||||
{
|
||||
var command = new DispatchWorkflowDefinitionCommand("definition-version-1")
|
||||
{
|
||||
InstanceId = "child-1",
|
||||
SkipIfInstanceExists = false
|
||||
};
|
||||
_workflowClient.InstanceExistsAsync(Arg.Any<CancellationToken>()).Returns(true);
|
||||
|
||||
await _handler.HandleAsync(command, CancellationToken.None);
|
||||
|
||||
await _workflowClient.DidNotReceive().InstanceExistsAsync(Arg.Any<CancellationToken>());
|
||||
await _workflowClient.Received(1).CreateAndRunInstanceAsync(
|
||||
Arg.Is<CreateAndRunWorkflowInstanceRequest>(x => x.WorkflowDefinitionHandle.DefinitionVersionId == "definition-version-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleAsync_DoesNotCreateAndRunWorkflow_WhenCommandInstanceAlreadyExistsAndIdempotencyIsRequested()
|
||||
{
|
||||
var command = new DispatchWorkflowDefinitionCommand("definition-version-1")
|
||||
{
|
||||
InstanceId = "child-1",
|
||||
SkipIfInstanceExists = true
|
||||
};
|
||||
_workflowClient.InstanceExistsAsync(Arg.Any<CancellationToken>()).Returns(true);
|
||||
|
||||
await _handler.HandleAsync(command, CancellationToken.None);
|
||||
|
||||
await _workflowClient.Received(1).InstanceExistsAsync(Arg.Any<CancellationToken>());
|
||||
await _workflowClient.DidNotReceiveWithAnyArgs().CreateAndRunInstanceAsync(default!, default);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleAsync_CreatesAndRunsWorkflow_WhenIdempotencyIsRequestedAndCommandInstanceDoesNotExist()
|
||||
{
|
||||
var command = new DispatchWorkflowDefinitionCommand("definition-version-1")
|
||||
{
|
||||
InstanceId = "child-1",
|
||||
SkipIfInstanceExists = true
|
||||
};
|
||||
_workflowClient.InstanceExistsAsync(Arg.Any<CancellationToken>()).Returns(false);
|
||||
|
||||
await _handler.HandleAsync(command, CancellationToken.None);
|
||||
|
||||
await _workflowClient.Received(1).InstanceExistsAsync(Arg.Any<CancellationToken>());
|
||||
await _workflowClient.Received(1).CreateAndRunInstanceAsync(
|
||||
Arg.Is<CreateAndRunWorkflowInstanceRequest>(x => x.WorkflowDefinitionHandle.DefinitionVersionId == "definition-version-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
public async Task HandleAsync_DoesNotCheckExistingInstance_WhenCommandInstanceIdIsMissing(string? instanceId)
|
||||
{
|
||||
var command = new DispatchWorkflowDefinitionCommand("definition-version-1")
|
||||
{
|
||||
InstanceId = instanceId,
|
||||
SkipIfInstanceExists = true
|
||||
};
|
||||
_workflowRuntime.CreateClientAsync(instanceId, Arg.Any<CancellationToken>()).Returns(new ValueTask<IWorkflowClient>(_workflowClient));
|
||||
|
||||
await _handler.HandleAsync(command, CancellationToken.None);
|
||||
|
||||
await _workflowClient.DidNotReceive().InstanceExistsAsync(Arg.Any<CancellationToken>());
|
||||
await _workflowClient.Received(1).CreateAndRunInstanceAsync(
|
||||
Arg.Is<CreateAndRunWorkflowInstanceRequest>(x => x.WorkflowDefinitionHandle.DefinitionVersionId == "definition-version-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
using Elsa.Workflows.Management.Entities;
|
||||
using Elsa.Workflows.Runtime.Handlers;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
using Elsa.Workflows.Runtime.Notifications;
|
||||
using Elsa.Workflows.Runtime.Options;
|
||||
using Elsa.Workflows.State;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.UnitTests.Handlers;
|
||||
|
||||
public class ProcessWorkflowDispatchOutboxTests
|
||||
{
|
||||
private readonly IWorkflowDispatchOutboxProcessor _processor = Substitute.For<IWorkflowDispatchOutboxProcessor>();
|
||||
private readonly WorkflowDispatcherOptions _options = new()
|
||||
{
|
||||
UseTransactionalOutbox = true,
|
||||
ProcessOutboxAfterCommit = true
|
||||
};
|
||||
private readonly IServiceProvider _serviceProvider = Substitute.For<IServiceProvider>();
|
||||
|
||||
public ProcessWorkflowDispatchOutboxTests()
|
||||
{
|
||||
_serviceProvider.GetService(typeof(IWorkflowDispatchOutboxProcessor)).Returns(_processor);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleAsync_DoesNotProcess_WhenCommittedStateHasNoOutboxItems()
|
||||
{
|
||||
var handler = CreateHandler();
|
||||
var notification = CreateNotification(new WorkflowState());
|
||||
|
||||
await handler.HandleAsync(notification, CancellationToken.None);
|
||||
|
||||
await _processor.DidNotReceive().TryProcessAsync(Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleAsync_Processes_WhenCommittedStateHasOutboxItems()
|
||||
{
|
||||
var handler = CreateHandler();
|
||||
var notification = CreateNotification(new WorkflowState
|
||||
{
|
||||
Properties = new Dictionary<string, object>
|
||||
{
|
||||
[WorkflowDispatchOutboxStateExtensions.PropertyKey] = new WorkflowDispatchOutboxState
|
||||
{
|
||||
ItemIds = ["outbox-1"]
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await handler.HandleAsync(notification, CancellationToken.None);
|
||||
|
||||
await _processor.Received(1).TryProcessAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleAsync_DoesNotThrow_WhenEagerOutboxProcessingFails()
|
||||
{
|
||||
var handler = CreateHandler();
|
||||
var notification = CreateNotification(new WorkflowState
|
||||
{
|
||||
Properties = new Dictionary<string, object>
|
||||
{
|
||||
[WorkflowDispatchOutboxStateExtensions.PropertyKey] = new WorkflowDispatchOutboxState
|
||||
{
|
||||
ItemIds = ["outbox-1"]
|
||||
}
|
||||
}
|
||||
});
|
||||
_processor.TryProcessAsync(Arg.Any<CancellationToken>()).Returns(Task.FromException<bool>(new InvalidOperationException("Store unavailable.")));
|
||||
|
||||
await handler.HandleAsync(notification, CancellationToken.None);
|
||||
}
|
||||
|
||||
private ProcessWorkflowDispatchOutbox CreateHandler() => new(_serviceProvider, Microsoft.Extensions.Options.Options.Create(_options), NullLogger<ProcessWorkflowDispatchOutbox>.Instance);
|
||||
|
||||
private static WorkflowStateCommitted CreateNotification(WorkflowState workflowState)
|
||||
{
|
||||
return new(null!, workflowState, new WorkflowInstance());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
using Elsa.KeyValues.Entities;
|
||||
using Elsa.KeyValues.Models;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.UnitTests.Models;
|
||||
|
||||
public class KeyValueFilterTests
|
||||
{
|
||||
[Fact]
|
||||
public void Apply_OrdersByPersistedKey_WhenOrderByKeyIsEnabled()
|
||||
{
|
||||
var filter = new KeyValueFilter { OrderByKey = true };
|
||||
|
||||
var result = filter.Apply(CreateUnorderedPairs()).Select(x => x.Key).ToList();
|
||||
|
||||
Assert.Equal(["key-a", "key-b", "key-c"], result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Apply_AppliesTakeAfterOrderingByPersistedKey()
|
||||
{
|
||||
var filter = new KeyValueFilter { OrderByKey = true, Take = 2 };
|
||||
|
||||
var result = filter.Apply(CreateUnorderedPairs()).Select(x => x.Key).ToList();
|
||||
|
||||
Assert.Equal(["key-a", "key-b"], result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Apply_ReturnsNoRecords_WhenTakeIsZero()
|
||||
{
|
||||
var filter = new KeyValueFilter { Take = 0 };
|
||||
|
||||
var result = filter.Apply(CreateUnorderedPairs()).ToList();
|
||||
|
||||
Assert.Empty(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Apply_DoesNotLimitRecords_WhenTakeIsNull()
|
||||
{
|
||||
var filter = new KeyValueFilter { Take = null };
|
||||
|
||||
var result = filter.Apply(CreateUnorderedPairs()).Select(x => x.Key).ToList();
|
||||
|
||||
Assert.Equal(["key-c", "key-a", "key-b"], result);
|
||||
}
|
||||
|
||||
private static IQueryable<SerializedKeyValuePair> CreateUnorderedPairs()
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
new SerializedKeyValuePair { Key = "key-c", SerializedValue = "c" },
|
||||
new SerializedKeyValuePair { Key = "key-a", SerializedValue = "a" },
|
||||
new SerializedKeyValuePair { Key = "key-b", SerializedValue = "b" }
|
||||
}.AsQueryable();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
using Elsa.Common.Services;
|
||||
using Elsa.Features.Services;
|
||||
using Elsa.KeyValues.Contracts;
|
||||
using Elsa.KeyValues.Entities;
|
||||
using Elsa.KeyValues.Models;
|
||||
using Elsa.KeyValues.Stores;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using NSubstitute;
|
||||
using ModuleKeyValueFeature = Elsa.KeyValues.Features.KeyValueFeature;
|
||||
using ShellKeyValueFeature = Elsa.KeyValues.ShellFeatures.KeyValueFeature;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.UnitTests.Services;
|
||||
|
||||
public class KeyValueFeatureTests
|
||||
{
|
||||
private readonly IServiceCollection _services = new ServiceCollection();
|
||||
private readonly IModule _module = Substitute.For<IModule>();
|
||||
|
||||
public KeyValueFeatureTests()
|
||||
{
|
||||
_module.Services.Returns(_services);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModuleFeature_DoesNotRegisterMemoryStore_WhenKeyValueStoreIsCustomized()
|
||||
{
|
||||
var feature = new ModuleKeyValueFeature(_module)
|
||||
{
|
||||
KeyValueStore = _ => new CustomKeyValueStore()
|
||||
};
|
||||
|
||||
feature.Apply();
|
||||
|
||||
Assert.DoesNotContain(_services, x => x.ServiceType == typeof(MemoryStore<SerializedKeyValuePair>));
|
||||
Assert.Contains(_services, x => x.ServiceType == typeof(IKeyValueStore));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModuleFeature_CanExplicitlyRegisterMemoryStore_WhenKeyValueStoreIsCustomized()
|
||||
{
|
||||
var feature = new ModuleKeyValueFeature(_module)
|
||||
{
|
||||
KeyValueStore = sp => ActivatorUtilities.CreateInstance<MemoryKeyValueStore>(sp),
|
||||
RegisterMemoryStore = true
|
||||
};
|
||||
|
||||
feature.Apply();
|
||||
|
||||
Assert.Contains(_services, x => x.ServiceType == typeof(MemoryStore<SerializedKeyValuePair>));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModuleFeature_ConfiguredKeyValueStoreOverridesExistingRegistration()
|
||||
{
|
||||
_services.AddScoped<IKeyValueStore>(_ => new ExistingKeyValueStore());
|
||||
var feature = new ModuleKeyValueFeature(_module)
|
||||
{
|
||||
KeyValueStore = _ => new CustomKeyValueStore()
|
||||
};
|
||||
|
||||
feature.Apply();
|
||||
|
||||
using var provider = _services.BuildServiceProvider();
|
||||
var store = provider.GetRequiredService<IKeyValueStore>();
|
||||
Assert.IsType<CustomKeyValueStore>(store);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShellFeature_DoesNotRegisterMemoryStore_WhenKeyValueStoreIsCustomized()
|
||||
{
|
||||
var feature = new ShellKeyValueFeature
|
||||
{
|
||||
KeyValueStore = _ => new CustomKeyValueStore()
|
||||
};
|
||||
|
||||
feature.ConfigureServices(_services);
|
||||
|
||||
Assert.DoesNotContain(_services, x => x.ServiceType == typeof(MemoryStore<SerializedKeyValuePair>));
|
||||
Assert.Contains(_services, x => x.ServiceType == typeof(IKeyValueStore));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShellFeature_CanExplicitlyRegisterMemoryStore_WhenKeyValueStoreIsCustomized()
|
||||
{
|
||||
var feature = new ShellKeyValueFeature
|
||||
{
|
||||
KeyValueStore = sp => ActivatorUtilities.CreateInstance<MemoryKeyValueStore>(sp),
|
||||
RegisterMemoryStore = true
|
||||
};
|
||||
|
||||
feature.ConfigureServices(_services);
|
||||
|
||||
Assert.Contains(_services, x => x.ServiceType == typeof(MemoryStore<SerializedKeyValuePair>));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShellFeature_ConfiguredKeyValueStoreOverridesExistingRegistration()
|
||||
{
|
||||
_services.AddScoped<IKeyValueStore>(_ => new ExistingKeyValueStore());
|
||||
var feature = new ShellKeyValueFeature
|
||||
{
|
||||
KeyValueStore = _ => new CustomKeyValueStore()
|
||||
};
|
||||
|
||||
feature.ConfigureServices(_services);
|
||||
|
||||
using var provider = _services.BuildServiceProvider();
|
||||
var store = provider.GetRequiredService<IKeyValueStore>();
|
||||
Assert.IsType<CustomKeyValueStore>(store);
|
||||
}
|
||||
|
||||
private class ExistingKeyValueStore : TestKeyValueStore;
|
||||
|
||||
private class CustomKeyValueStore : TestKeyValueStore;
|
||||
|
||||
private abstract class TestKeyValueStore : IKeyValueStore
|
||||
{
|
||||
public Task SaveAsync(SerializedKeyValuePair keyValuePair, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task<SerializedKeyValuePair?> FindAsync(KeyValueFilter filter, CancellationToken cancellationToken) => Task.FromResult<SerializedKeyValuePair?>(null);
|
||||
|
||||
public Task<IEnumerable<SerializedKeyValuePair>> FindManyAsync(KeyValueFilter filter, CancellationToken cancellationToken) => Task.FromResult<IEnumerable<SerializedKeyValuePair>>([]);
|
||||
|
||||
public Task DeleteAsync(string key, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,447 @@
|
|||
using System.Text.Json;
|
||||
using Elsa.Common.Services;
|
||||
using Elsa.KeyValues.Contracts;
|
||||
using Elsa.KeyValues.Entities;
|
||||
using Elsa.KeyValues.Models;
|
||||
using Elsa.KeyValues.Stores;
|
||||
using Elsa.Workflows.Runtime.Commands;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
using NSubstitute;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.UnitTests.Services;
|
||||
|
||||
public class KeyValueWorkflowDispatchOutboxStoreTests
|
||||
{
|
||||
private readonly IKeyValueStore _keyValueStore = Substitute.For<IKeyValueStore>();
|
||||
private readonly TestPayloadSerializer _payloadSerializer = new();
|
||||
private readonly KeyValueWorkflowDispatchOutboxStore _store;
|
||||
|
||||
public KeyValueWorkflowDispatchOutboxStoreTests()
|
||||
{
|
||||
_store = new(_keyValueStore, _payloadSerializer);
|
||||
_keyValueStore.FindAsync(Arg.Any<KeyValueFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<SerializedKeyValuePair?>(null));
|
||||
_keyValueStore.FindManyAsync(Arg.Any<KeyValueFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Array.Empty<SerializedKeyValuePair>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_ReturnsOldestItemsBeforeApplyingLimit()
|
||||
{
|
||||
var newest = CreateItem("newest", new DateTimeOffset(2026, 5, 20, 12, 2, 0, TimeSpan.Zero));
|
||||
var oldest = CreateItem("oldest", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var middle = CreateItem("middle", new DateTimeOffset(2026, 5, 20, 12, 1, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items = new[] { newest, oldest, middle }.ToDictionary(x => x.Id);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:"), Arg.Any<CancellationToken>())
|
||||
.Returns(new[] { oldest, middle }.Select(x => new SerializedKeyValuePair { Key = $"index:{x.Id}", SerializedValue = x.Id }));
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Keys != null), Arg.Any<CancellationToken>())
|
||||
.Returns(callInfo => ((KeyValueFilter)callInfo[0]).Keys!.Select(x => new SerializedKeyValuePair { Key = x, SerializedValue = x.Split(':').Last() }));
|
||||
|
||||
var result = (await _store.FindManyAsync(2)).ToList();
|
||||
|
||||
Assert.Equal(["oldest", "middle"], result.Select(x => x.Id));
|
||||
await _keyValueStore.Received(1).FindManyAsync(
|
||||
Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:" && x.OrderByKey && x.Take == 2),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_MergesIndexedLegacyAndOrphanItems()
|
||||
{
|
||||
var indexed = CreateItem("indexed", new DateTimeOffset(2026, 5, 20, 12, 2, 0, TimeSpan.Zero));
|
||||
var legacy = CreateItem("legacy", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var orphan = CreateItem("orphan", new DateTimeOffset(2026, 5, 20, 12, 1, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items = new[] { indexed, legacy, orphan }.ToDictionary(x => x.Id);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:"), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateIndexRecord(indexed)]);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Keys != null), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateItemRecord(indexed)]);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateItemRecord(indexed), CreateLegacyRecord(legacy), CreateItemRecord(orphan), CreateIndexRecord(indexed)]);
|
||||
|
||||
var result = (await _store.FindManyAsync()).ToList();
|
||||
|
||||
Assert.Equal(["legacy", "orphan", "indexed"], result.Select(x => x.Id));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_AppliesLimitAfterMergingRecoverableItems()
|
||||
{
|
||||
var indexed1 = CreateItem("indexed-1", new DateTimeOffset(2026, 5, 20, 12, 1, 0, TimeSpan.Zero));
|
||||
var indexed2 = CreateItem("indexed-2", new DateTimeOffset(2026, 5, 20, 12, 2, 0, TimeSpan.Zero));
|
||||
var orphan = CreateItem("orphan", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items = new[] { indexed1, indexed2, orphan }.ToDictionary(x => x.Id);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:"), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateIndexRecord(indexed1), CreateIndexRecord(indexed2)]);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Keys != null), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateItemRecord(indexed1), CreateItemRecord(indexed2)]);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateItemRecord(orphan)]);
|
||||
|
||||
var result = (await _store.FindManyAsync(2)).ToList();
|
||||
|
||||
Assert.Equal(["orphan", "indexed-1"], result.Select(x => x.Id));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_MigratesOnlyCallerLimitFromLegacyScan()
|
||||
{
|
||||
var first = CreateItem("legacy-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var second = CreateItem("legacy-2", new DateTimeOffset(2026, 5, 20, 12, 1, 0, TimeSpan.Zero));
|
||||
var third = CreateItem("legacy-3", new DateTimeOffset(2026, 5, 20, 12, 2, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items = new[] { first, second, third }.ToDictionary(x => x.Id);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateLegacyRecord(first), CreateLegacyRecord(second), CreateLegacyRecord(third)]);
|
||||
|
||||
var result = (await _store.FindManyAsync(2)).ToList();
|
||||
|
||||
Assert.Equal(["legacy-1", "legacy-2"], result.Select(x => x.Id));
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:legacy-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:legacy-2"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.DidNotReceive().SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:legacy-3"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.DidNotReceive().SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:State:LegacyScanCompleted"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).FindManyAsync(
|
||||
Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:" && x.StartsWith && x.OrderByKey && x.Take == 3),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_MarksLegacyScanCompleted_WhenCallerLimitCoversRecoverableItems()
|
||||
{
|
||||
var first = CreateItem("legacy-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var second = CreateItem("legacy-2", new DateTimeOffset(2026, 5, 20, 12, 1, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items = new[] { first, second }.ToDictionary(x => x.Id);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateLegacyRecord(first), CreateLegacyRecord(second)]);
|
||||
|
||||
var result = (await _store.FindManyAsync(2)).ToList();
|
||||
|
||||
Assert.Equal(["legacy-1", "legacy-2"], result.Select(x => x.Id));
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:State:LegacyScanCompleted" && x.SerializedValue == "true"),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_UsesRecoveryMarkersWithoutBroadScan_WhenLegacyScanAlreadyCompleted()
|
||||
{
|
||||
var orphan = CreateItem("orphan", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items[orphan.Id] = orphan;
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:State:LegacyScanCompleted"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:State:LegacyScanCompleted", SerializedValue = "true" });
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Recovery:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateRecoveryRecord(orphan)]);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Keys != null), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateItemRecord(orphan)]);
|
||||
|
||||
var result = (await _store.FindManyAsync()).ToList();
|
||||
|
||||
Assert.Equal(["orphan"], result.Select(x => x.Id));
|
||||
await _keyValueStore.DidNotReceive().FindManyAsync(
|
||||
Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:" && x.StartsWith),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == $"Elsa:WorkflowDispatchOutbox:Index:{orphan.CreatedAt.UtcTicks:D20}:orphan" && x.SerializedValue == "orphan"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:Recovery:orphan", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_AppliesCallerLimitToRecoveryScan()
|
||||
{
|
||||
var first = CreateItem("orphan-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var second = CreateItem("orphan-2", new DateTimeOffset(2026, 5, 20, 12, 1, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items = new[] { first, second }.ToDictionary(x => x.Id);
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:State:LegacyScanCompleted"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:State:LegacyScanCompleted", SerializedValue = "true" });
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Recovery:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateRecoveryRecord(first), CreateRecoveryRecord(second)]);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Keys != null), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateItemRecord(first), CreateItemRecord(second)]);
|
||||
|
||||
var result = (await _store.FindManyAsync(2)).ToList();
|
||||
|
||||
Assert.Equal(["orphan-1", "orphan-2"], result.Select(x => x.Id));
|
||||
await _keyValueStore.Received(1).FindManyAsync(
|
||||
Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Recovery:" && x.StartsWith && x.OrderByKey && x.Take == 2),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_CleansUpCorruptIndexedItem_AllowingLaterItemsToProgress()
|
||||
{
|
||||
var keyValueStore = new MemoryKeyValueStore(new MemoryStore<SerializedKeyValuePair>());
|
||||
var payloadSerializer = new TestPayloadSerializer();
|
||||
var store = new KeyValueWorkflowDispatchOutboxStore(keyValueStore, payloadSerializer);
|
||||
var corrupt = CreateItem("corrupt", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var valid = CreateItem("valid", new DateTimeOffset(2026, 5, 20, 12, 1, 0, TimeSpan.Zero));
|
||||
payloadSerializer.Items[valid.Id] = valid;
|
||||
await SaveAsync(keyValueStore, CreateItemRecord(corrupt));
|
||||
await SaveAsync(keyValueStore, CreateIndexRecord(corrupt));
|
||||
await SaveAsync(keyValueStore, CreateIndexByIdRecord(corrupt));
|
||||
await SaveAsync(keyValueStore, CreateRecoveryRecord(corrupt));
|
||||
await SaveAsync(keyValueStore, CreateItemRecord(valid));
|
||||
await SaveAsync(keyValueStore, CreateIndexRecord(valid));
|
||||
await SaveAsync(keyValueStore, CreateIndexByIdRecord(valid));
|
||||
await SaveAsync(keyValueStore, CreateLegacyScanCompletedRecord());
|
||||
|
||||
var firstBatch = (await store.FindManyAsync(1)).ToList();
|
||||
var secondBatch = (await store.FindManyAsync(1)).ToList();
|
||||
|
||||
Assert.Empty(firstBatch);
|
||||
Assert.Equal(["valid"], secondBatch.Select(x => x.Id));
|
||||
Assert.Null(await FindAsync(keyValueStore, $"Elsa:WorkflowDispatchOutbox:Items:{corrupt.Id}"));
|
||||
Assert.Null(await FindAsync(keyValueStore, $"Elsa:WorkflowDispatchOutbox:Index:{corrupt.CreatedAt.UtcTicks:D20}:{corrupt.Id}"));
|
||||
Assert.Null(await FindAsync(keyValueStore, $"Elsa:WorkflowDispatchOutbox:IndexById:{corrupt.Id}"));
|
||||
Assert.Null(await FindAsync(keyValueStore, $"Elsa:WorkflowDispatchOutbox:Recovery:{corrupt.Id}"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindManyAsync_CleansUpRecoveryRecord_WhenPayloadCannotBeDeserialized()
|
||||
{
|
||||
var item = CreateItem("corrupt", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var indexKey = $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:{item.Id}";
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Recovery:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateRecoveryRecord(item)]);
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Keys != null), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateItemRecord(item)]);
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == $"Elsa:WorkflowDispatchOutbox:IndexById:{item.Id}"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = $"Elsa:WorkflowDispatchOutbox:IndexById:{item.Id}", SerializedValue = indexKey });
|
||||
|
||||
var result = (await _store.FindManyAsync()).ToList();
|
||||
|
||||
Assert.Empty(result);
|
||||
await _keyValueStore.Received(1).DeleteAsync(indexKey, Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync($"Elsa:WorkflowDispatchOutbox:IndexById:{item.Id}", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync($"Elsa:WorkflowDispatchOutbox:Recovery:{item.Id}", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync($"Elsa:WorkflowDispatchOutbox:Items:{item.Id}", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SaveAsync_WritesItemAndSortableIndex()
|
||||
{
|
||||
var item = CreateItem("outbox-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
item.TenantId = "tenant-1";
|
||||
|
||||
await _store.SaveAsync(item);
|
||||
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Recovery:outbox-1" && x.SerializedValue == "outbox-1" && x.TenantId == "tenant-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:outbox-1" && x.SerializedValue == "outbox-1" && x.TenantId == "tenant-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1" && x.SerializedValue == "outbox-1" && x.TenantId == "tenant-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).SaveAsync(
|
||||
Arg.Is<SerializedKeyValuePair>(x => x.Key == "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1" && x.SerializedValue == $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1" && x.TenantId == "tenant-1"),
|
||||
Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:Recovery:outbox-1", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_RemovesIndexBeforeItem()
|
||||
{
|
||||
var item = CreateItem("outbox-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
_payloadSerializer.Items[item.Id] = item;
|
||||
_keyValueStore.FindAsync(Arg.Any<KeyValueFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:Items:outbox-1", SerializedValue = item.Id });
|
||||
|
||||
await _store.DeleteAsync(item.Id);
|
||||
|
||||
Received.InOrder(() =>
|
||||
{
|
||||
_keyValueStore.DeleteAsync($"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1", Arg.Any<CancellationToken>());
|
||||
_keyValueStore.DeleteAsync("Elsa:WorkflowDispatchOutbox:Items:outbox-1", Arg.Any<CancellationToken>());
|
||||
});
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:Items:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync($"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:Recovery:outbox-1", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_CleansUpIndexRecords_WhenItemRecordIsMissing()
|
||||
{
|
||||
var item = CreateItem("outbox-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<SerializedKeyValuePair?>(null));
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<SerializedKeyValuePair?>(null));
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateIndexRecord(item), CreateIndexRecord(CreateItem("outbox-2", item.CreatedAt))]);
|
||||
|
||||
await _store.DeleteAsync(item.Id);
|
||||
|
||||
await _keyValueStore.Received(1).DeleteAsync($"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.DidNotReceive().DeleteAsync($"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-2", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:Items:outbox-1", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_CleansUpIndexRecords_WhenItemPayloadCannotBeDeserialized()
|
||||
{
|
||||
var item = CreateItem("outbox-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var indexKey = $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1";
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:Items:outbox-1", SerializedValue = "invalid" });
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", SerializedValue = indexKey });
|
||||
|
||||
await _store.DeleteAsync(item.Id);
|
||||
|
||||
await _keyValueStore.Received(1).DeleteAsync(indexKey, Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:Items:outbox-1", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_UsesIndexByIdBeforeScanningAllIndexes()
|
||||
{
|
||||
var item = CreateItem("outbox-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var indexKey = $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1";
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<SerializedKeyValuePair?>(null));
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", SerializedValue = indexKey });
|
||||
|
||||
await _store.DeleteAsync(item.Id);
|
||||
|
||||
await _keyValueStore.Received(1).DeleteAsync(indexKey, Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.DidNotReceive().FindManyAsync(
|
||||
Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:" && x.StartsWith),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_UsesScan_WhenIndexByIdValueDoesNotMatchExpectedId()
|
||||
{
|
||||
var item = CreateItem("outbox-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
var mismatchedIndexKey = $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-2";
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<SerializedKeyValuePair?>(null));
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", SerializedValue = mismatchedIndexKey });
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateIndexRecord(item)]);
|
||||
|
||||
await _store.DeleteAsync(item.Id);
|
||||
|
||||
await _keyValueStore.DidNotReceive().DeleteAsync(mismatchedIndexKey, Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync($"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).FindManyAsync(
|
||||
Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:" && x.StartsWith),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_UsesScan_WhenIndexByIdValueIsNotWellFormed()
|
||||
{
|
||||
var item = CreateItem("outbox-1", new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero));
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Items:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<SerializedKeyValuePair?>(null));
|
||||
_keyValueStore.FindAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1"), Arg.Any<CancellationToken>())
|
||||
.Returns(new SerializedKeyValuePair { Key = "Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", SerializedValue = "arbitrary-key" });
|
||||
_keyValueStore.FindManyAsync(Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:" && x.StartsWith), Arg.Any<CancellationToken>())
|
||||
.Returns([CreateIndexRecord(item)]);
|
||||
|
||||
await _store.DeleteAsync(item.Id);
|
||||
|
||||
await _keyValueStore.DidNotReceive().DeleteAsync("arbitrary-key", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync($"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).DeleteAsync("Elsa:WorkflowDispatchOutbox:IndexById:outbox-1", Arg.Any<CancellationToken>());
|
||||
await _keyValueStore.Received(1).FindManyAsync(
|
||||
Arg.Is<KeyValueFilter>(x => x.Key == "Elsa:WorkflowDispatchOutbox:Index:" && x.StartsWith),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
private static WorkflowDispatchOutboxItem CreateItem(string id, DateTimeOffset createdAt)
|
||||
{
|
||||
return new()
|
||||
{
|
||||
Id = id,
|
||||
OwnerWorkflowInstanceId = "parent-1",
|
||||
Kind = WorkflowDispatchOutboxItemKind.WorkflowDefinition,
|
||||
WorkflowDefinitionCommand = new DispatchWorkflowDefinitionCommand("definition-version-1"),
|
||||
CreatedAt = createdAt
|
||||
};
|
||||
}
|
||||
|
||||
private static SerializedKeyValuePair CreateItemRecord(WorkflowDispatchOutboxItem item) => new()
|
||||
{
|
||||
Key = $"Elsa:WorkflowDispatchOutbox:Items:{item.Id}",
|
||||
SerializedValue = item.Id
|
||||
};
|
||||
|
||||
private static SerializedKeyValuePair CreateLegacyRecord(WorkflowDispatchOutboxItem item) => new()
|
||||
{
|
||||
Key = $"Elsa:WorkflowDispatchOutbox:{item.Id}",
|
||||
SerializedValue = item.Id
|
||||
};
|
||||
|
||||
private static SerializedKeyValuePair CreateIndexRecord(WorkflowDispatchOutboxItem item) => new()
|
||||
{
|
||||
Key = $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:{item.Id}",
|
||||
SerializedValue = item.Id
|
||||
};
|
||||
|
||||
private static SerializedKeyValuePair CreateIndexByIdRecord(WorkflowDispatchOutboxItem item) => new()
|
||||
{
|
||||
Key = $"Elsa:WorkflowDispatchOutbox:IndexById:{item.Id}",
|
||||
SerializedValue = $"Elsa:WorkflowDispatchOutbox:Index:{item.CreatedAt.UtcTicks:D20}:{item.Id}"
|
||||
};
|
||||
|
||||
private static SerializedKeyValuePair CreateRecoveryRecord(WorkflowDispatchOutboxItem item) => new()
|
||||
{
|
||||
Key = $"Elsa:WorkflowDispatchOutbox:Recovery:{item.Id}",
|
||||
SerializedValue = item.Id
|
||||
};
|
||||
|
||||
private static SerializedKeyValuePair CreateLegacyScanCompletedRecord() => new()
|
||||
{
|
||||
Key = "Elsa:WorkflowDispatchOutbox:State:LegacyScanCompleted",
|
||||
SerializedValue = "true"
|
||||
};
|
||||
|
||||
private static async Task SaveAsync(IKeyValueStore keyValueStore, SerializedKeyValuePair record)
|
||||
{
|
||||
await keyValueStore.SaveAsync(record, CancellationToken.None);
|
||||
}
|
||||
|
||||
private static async Task<SerializedKeyValuePair?> FindAsync(IKeyValueStore keyValueStore, string key)
|
||||
{
|
||||
return await keyValueStore.FindAsync(new KeyValueFilter { Key = key }, CancellationToken.None);
|
||||
}
|
||||
|
||||
private class TestPayloadSerializer : IPayloadSerializer
|
||||
{
|
||||
public IDictionary<string, WorkflowDispatchOutboxItem> Items { get; set; } = new Dictionary<string, WorkflowDispatchOutboxItem>();
|
||||
|
||||
public string Serialize(object payload) => ((WorkflowDispatchOutboxItem)payload).Id;
|
||||
|
||||
public JsonElement SerializeToElement(object payload) => throw new NotSupportedException();
|
||||
|
||||
public object Deserialize(string serializedData) => Deserialize<WorkflowDispatchOutboxItem>(serializedData);
|
||||
|
||||
public object Deserialize(string serializedData, Type type) => Deserialize(serializedData);
|
||||
|
||||
public object Deserialize(JsonElement serializedData) => throw new NotSupportedException();
|
||||
|
||||
public T Deserialize<T>(string serializedData) => Items.TryGetValue(serializedData, out var item) ? (T)(object)item : default!;
|
||||
|
||||
public T Deserialize<T>(JsonElement serializedData) => throw new NotSupportedException();
|
||||
|
||||
public JsonSerializerOptions GetOptions() => new();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,558 @@
|
|||
using Elsa.Common;
|
||||
using Elsa.Common.DistributedHosting;
|
||||
using Elsa.Common.DistributedHosting.DistributedLocks;
|
||||
using Elsa.Common.Multitenancy;
|
||||
using Elsa.Mediator;
|
||||
using Elsa.Mediator.Contracts;
|
||||
using Elsa.Workflows.Management;
|
||||
using Elsa.Workflows.Management.Entities;
|
||||
using Elsa.Workflows.Management.Filters;
|
||||
using Elsa.Workflows.Runtime.Commands;
|
||||
using Elsa.Workflows.Runtime.Models;
|
||||
using Elsa.Workflows.Runtime.Options;
|
||||
using Elsa.Workflows.State;
|
||||
using Medallion.Threading;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
|
||||
namespace Elsa.Workflows.Runtime.UnitTests.Services;
|
||||
|
||||
public class WorkflowDispatchOutboxProcessorTests
|
||||
{
|
||||
private readonly IWorkflowDispatchOutboxStore _store = Substitute.For<IWorkflowDispatchOutboxStore>();
|
||||
private readonly IWorkflowInstanceStore _workflowInstanceStore = Substitute.For<IWorkflowInstanceStore>();
|
||||
private readonly ICommandSender _commandSender = Substitute.For<ICommandSender>();
|
||||
private readonly ISystemClock _systemClock = Substitute.For<ISystemClock>();
|
||||
private readonly WorkflowDispatcherOptions _options = new();
|
||||
private readonly WorkflowDispatchOutboxProcessor _processor;
|
||||
|
||||
public WorkflowDispatchOutboxProcessorTests()
|
||||
{
|
||||
_systemClock.UtcNow.Returns(DateTimeOffset.UtcNow);
|
||||
_processor = CreateProcessor();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_ReturnsWithoutScanning_WhenProcessorLockCannotBeAcquired()
|
||||
{
|
||||
var processor = CreateProcessor(new ContendedDistributedSynchronizationProvider());
|
||||
|
||||
await processor.ProcessAsync();
|
||||
|
||||
await _store.DidNotReceiveWithAnyArgs().FindManyAsync(default, default);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_ReturnsWithoutScanning_WhenProcessorLockAcquisitionTimesOut()
|
||||
{
|
||||
var processor = CreateProcessor(new TimeoutDistributedSynchronizationProvider());
|
||||
|
||||
await processor.ProcessAsync();
|
||||
|
||||
await _store.DidNotReceiveWithAnyArgs().FindManyAsync(default, default);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryProcessAsync_ReturnsFalseWithoutScanning_WhenProcessorLockAcquisitionTimesOut()
|
||||
{
|
||||
var processor = CreateProcessor(new TimeoutDistributedSynchronizationProvider());
|
||||
|
||||
var result = await processor.TryProcessAsync();
|
||||
|
||||
Assert.False(result);
|
||||
await _store.DidNotReceiveWithAnyArgs().FindManyAsync(default, default);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_UsesTenantScopedProcessorLock_WhenTenantContextIsAvailable()
|
||||
{
|
||||
var lockProvider = new RecordingDistributedSynchronizationProvider();
|
||||
var tenantAccessor = new DefaultTenantAccessor();
|
||||
var processor = CreateProcessor(lockProvider, tenantAccessor);
|
||||
using var tenantScope = tenantAccessor.PushContext(new Tenant { Id = "tenant-a" });
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(_ =>
|
||||
{
|
||||
Assert.Equal("Elsa:WorkflowDispatchOutbox:Processor:tenant-a", lockProvider.CurrentLockName);
|
||||
return [];
|
||||
});
|
||||
|
||||
await processor.ProcessAsync();
|
||||
|
||||
await _store.Received(1).FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryProcessAsync_UsesTenantScopedProcessorLock_WhenTenantContextIsAvailable()
|
||||
{
|
||||
var lockProvider = new RecordingDistributedSynchronizationProvider();
|
||||
var tenantAccessor = new DefaultTenantAccessor();
|
||||
var processor = CreateProcessor(lockProvider, tenantAccessor);
|
||||
using var tenantScope = tenantAccessor.PushContext(new Tenant { Id = "tenant-a" });
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(_ =>
|
||||
{
|
||||
Assert.Equal("Elsa:WorkflowDispatchOutbox:Processor:tenant-a", lockProvider.CurrentLockName);
|
||||
return [];
|
||||
});
|
||||
|
||||
var result = await processor.TryProcessAsync();
|
||||
|
||||
Assert.True(result);
|
||||
await _store.Received(1).FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
private WorkflowDispatchOutboxProcessor CreateProcessor(IDistributedLockProvider? distributedLockProvider = null, ITenantAccessor? tenantAccessor = null)
|
||||
{
|
||||
return new(
|
||||
_store,
|
||||
_workflowInstanceStore,
|
||||
_commandSender,
|
||||
distributedLockProvider ?? new NoopDistributedSynchronizationProvider(),
|
||||
_systemClock,
|
||||
Microsoft.Extensions.Options.Options.Create(new DistributedLockingOptions()),
|
||||
Microsoft.Extensions.Options.Options.Create(_options),
|
||||
NullLogger<WorkflowDispatchOutboxProcessor>.Instance,
|
||||
tenantAccessor);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DoesNotDispatch_WhenOwnerWorkflowHasNotCommittedOutboxMarker()
|
||||
{
|
||||
var item = CreateItem();
|
||||
_systemClock.UtcNow.Returns(item.CreatedAt.Add(_options.OrphanedOutboxItemRetention).AddTicks(-1));
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: false)));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.DidNotReceiveWithAnyArgs().SendAsync(default(DispatchWorkflowDefinitionCommand)!, default!, default!, default);
|
||||
await _store.DidNotReceive().DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DeletesOutboxItem_WhenOwnerWorkflowNeverCommittedMarkerAfterRetentionPeriod()
|
||||
{
|
||||
var createdAt = new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero);
|
||||
var item = CreateItem(createdAt);
|
||||
_systemClock.UtcNow.Returns(createdAt.Add(_options.OrphanedOutboxItemRetention));
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: false)));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.DidNotReceiveWithAnyArgs().SendAsync(default(DispatchWorkflowDefinitionCommand)!, default!, default!, default);
|
||||
await _store.Received(1).DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DoesNotCountDeliveryFailure_WhenUncommittedCleanupDeleteFails()
|
||||
{
|
||||
var createdAt = new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero);
|
||||
var item = CreateItem(createdAt);
|
||||
_systemClock.UtcNow.Returns(createdAt.Add(_options.OrphanedOutboxItemRetention));
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: false)));
|
||||
_store.DeleteAsync(item.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromException(new InvalidOperationException("Delete failed.")));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
Assert.Equal(0, item.DeliveryAttempts);
|
||||
await _store.DidNotReceive().SaveAsync(item, Arg.Any<CancellationToken>());
|
||||
await _commandSender.DidNotReceiveWithAnyArgs().SendAsync(default(DispatchWorkflowDefinitionCommand)!, default!, default!, default);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_KeepsOutboxItem_WhenOwnerWorkflowIsMissingWithinRetentionPeriod()
|
||||
{
|
||||
var createdAt = new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero);
|
||||
var item = CreateItem(createdAt);
|
||||
_systemClock.UtcNow.Returns(createdAt.Add(_options.OrphanedOutboxItemRetention).AddTicks(-1));
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>((WorkflowInstance?)null));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.DidNotReceiveWithAnyArgs().SendAsync(default(DispatchWorkflowDefinitionCommand)!, default!, default!, default);
|
||||
await _store.DidNotReceive().DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DeletesOutboxItem_WhenOwnerWorkflowIsMissingAfterRetentionPeriod()
|
||||
{
|
||||
var createdAt = new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero);
|
||||
var item = CreateItem(createdAt);
|
||||
_systemClock.UtcNow.Returns(createdAt.Add(_options.OrphanedOutboxItemRetention));
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>((WorkflowInstance?)null));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.DidNotReceiveWithAnyArgs().SendAsync(default(DispatchWorkflowDefinitionCommand)!, default!, default!, default);
|
||||
await _store.Received(1).DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DoesNotCountDeliveryFailure_WhenMissingOwnerCleanupDeleteFails()
|
||||
{
|
||||
var createdAt = new DateTimeOffset(2026, 5, 20, 12, 0, 0, TimeSpan.Zero);
|
||||
var item = CreateItem(createdAt);
|
||||
_systemClock.UtcNow.Returns(createdAt.Add(_options.OrphanedOutboxItemRetention));
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>((WorkflowInstance?)null));
|
||||
_store.DeleteAsync(item.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromException(new InvalidOperationException("Delete failed.")));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
Assert.Equal(0, item.DeliveryAttempts);
|
||||
await _store.DidNotReceive().SaveAsync(item, Arg.Any<CancellationToken>());
|
||||
await _commandSender.DidNotReceiveWithAnyArgs().SendAsync(default(DispatchWorkflowDefinitionCommand)!, default!, default!, default);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DispatchesAndDeletes_WhenOwnerWorkflowCommittedOutboxMarker()
|
||||
{
|
||||
var item = CreateItem();
|
||||
var owner = CreateOwnerWorkflowInstance(includeOutboxMarker: true);
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(owner));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.Received(1).SendAsync(
|
||||
item.WorkflowDefinitionCommand!,
|
||||
CommandStrategy.Background,
|
||||
Arg.Any<IDictionary<object, object>>(),
|
||||
CancellationToken.None);
|
||||
await _store.Received(1).DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
await _workflowInstanceStore.Received(1).SaveAsync(
|
||||
Arg.Is<WorkflowInstance>(x => !x.WorkflowState.HasWorkflowDispatchOutboxItem(item.Id)),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_ReloadsOwnerBeforeRemovingCommittedMarker()
|
||||
{
|
||||
var item = CreateItem();
|
||||
var staleOwner = CreateOwnerWorkflowInstance(includeOutboxMarker: true);
|
||||
var freshOwner = CreateOwnerWorkflowInstance(includeOutboxMarker: true);
|
||||
freshOwner.Name = "fresh owner";
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(
|
||||
new ValueTask<WorkflowInstance?>(staleOwner),
|
||||
new ValueTask<WorkflowInstance?>(freshOwner));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _workflowInstanceStore.Received(1).SaveAsync(
|
||||
Arg.Is<WorkflowInstance>(x => ReferenceEquals(x, freshOwner) && !x.WorkflowState.HasWorkflowDispatchOutboxItem(item.Id)),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_SavesMarkerRemovalWhileHoldingOwnerWorkflowInstanceLock()
|
||||
{
|
||||
var lockProvider = new RecordingDistributedSynchronizationProvider();
|
||||
var processor = CreateProcessor(lockProvider);
|
||||
var item = CreateItem();
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true)));
|
||||
_workflowInstanceStore.SaveAsync(Arg.Any<WorkflowInstance>(), Arg.Any<CancellationToken>())
|
||||
.Returns(_ =>
|
||||
{
|
||||
Assert.Equal("workflow-instance:parent-1", lockProvider.CurrentLockName);
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
|
||||
await processor.ProcessAsync();
|
||||
|
||||
await _workflowInstanceStore.Received(1).SaveAsync(
|
||||
Arg.Is<WorkflowInstance>(x => !x.WorkflowState.HasWorkflowDispatchOutboxItem(item.Id)),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DoesNotRecreateOutboxItem_WhenMarkerCleanupFails()
|
||||
{
|
||||
var item = CreateItem();
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true)));
|
||||
_workflowInstanceStore.SaveAsync(Arg.Any<WorkflowInstance>(), Arg.Any<CancellationToken>())
|
||||
.Returns<ValueTask>(_ => throw new InvalidOperationException("Store unavailable."));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _store.Received(1).DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
await _store.DidNotReceive().SaveAsync(item, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DoesNotCountDeliveryFailure_WhenDeleteAfterSendFails()
|
||||
{
|
||||
var item = CreateItem();
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true)));
|
||||
_store.DeleteAsync(item.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromException(new InvalidOperationException("Delete failed.")));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.Received(1).SendAsync(
|
||||
item.WorkflowDefinitionCommand!,
|
||||
CommandStrategy.Background,
|
||||
Arg.Any<IDictionary<object, object>>(),
|
||||
CancellationToken.None);
|
||||
Assert.Equal(0, item.DeliveryAttempts);
|
||||
await _store.DidNotReceive().SaveAsync(item, Arg.Any<CancellationToken>());
|
||||
await _workflowInstanceStore.DidNotReceive().SaveAsync(Arg.Any<WorkflowInstance>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_KeepsOutboxItem_WhenDispatchFails()
|
||||
{
|
||||
var item = CreateItem();
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true)));
|
||||
_commandSender
|
||||
.SendAsync(item.WorkflowDefinitionCommand!, CommandStrategy.Background, Arg.Any<IDictionary<object, object>>(), CancellationToken.None)
|
||||
.Returns(Task.FromException<Elsa.Mediator.Models.Unit>(new InvalidOperationException("Queue unavailable.")));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _store.DidNotReceive().DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
await _store.Received(1).SaveAsync(Arg.Is<WorkflowDispatchOutboxItem>(x => x.Id == item.Id && x.DeliveryAttempts == 1), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_ContinuesBatch_WhenFailurePersistenceFails()
|
||||
{
|
||||
var failedItem = CreateItem(id: "outbox-1");
|
||||
var nextItem = CreateItem(id: "outbox-2");
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([failedItem, nextItem]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(
|
||||
new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true, itemIds: [failedItem.Id])),
|
||||
new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true, itemIds: [nextItem.Id])),
|
||||
new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true, itemIds: [nextItem.Id])));
|
||||
_commandSender
|
||||
.SendAsync(failedItem.WorkflowDefinitionCommand!, CommandStrategy.Background, Arg.Any<IDictionary<object, object>>(), CancellationToken.None)
|
||||
.Returns(Task.FromException<Elsa.Mediator.Models.Unit>(new InvalidOperationException("Queue unavailable.")));
|
||||
_store.SaveAsync(failedItem, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromException(new InvalidOperationException("Save failed.")));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.Received(1).SendAsync(
|
||||
nextItem.WorkflowDefinitionCommand!,
|
||||
CommandStrategy.Background,
|
||||
Arg.Any<IDictionary<object, object>>(),
|
||||
CancellationToken.None);
|
||||
await _store.Received(1).DeleteAsync(nextItem.Id, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_DeletesOutboxItem_WhenMaxDeliveryAttemptsIsReached()
|
||||
{
|
||||
var item = CreateItem();
|
||||
item.DeliveryAttempts = _options.MaxOutboxDeliveryAttempts - 1;
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([item]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true)));
|
||||
_commandSender
|
||||
.SendAsync(item.WorkflowDefinitionCommand!, CommandStrategy.Background, Arg.Any<IDictionary<object, object>>(), CancellationToken.None)
|
||||
.Returns(Task.FromException<Elsa.Mediator.Models.Unit>(new InvalidOperationException("Queue unavailable.")));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _store.Received(1).DeleteAsync(item.Id, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessAsync_ContinuesBatch_WhenMaxAttemptDeleteFails()
|
||||
{
|
||||
var failedItem = CreateItem(id: "outbox-1");
|
||||
var nextItem = CreateItem(id: "outbox-2");
|
||||
failedItem.DeliveryAttempts = _options.MaxOutboxDeliveryAttempts - 1;
|
||||
_store.FindManyAsync(Arg.Any<int>(), Arg.Any<CancellationToken>()).Returns([failedItem, nextItem]);
|
||||
_workflowInstanceStore.FindAsync(Arg.Any<WorkflowInstanceFilter>(), Arg.Any<CancellationToken>())
|
||||
.Returns(
|
||||
new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true, itemIds: [failedItem.Id])),
|
||||
new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true, itemIds: [nextItem.Id])),
|
||||
new ValueTask<WorkflowInstance?>(CreateOwnerWorkflowInstance(includeOutboxMarker: true, itemIds: [nextItem.Id])));
|
||||
_commandSender
|
||||
.SendAsync(failedItem.WorkflowDefinitionCommand!, CommandStrategy.Background, Arg.Any<IDictionary<object, object>>(), CancellationToken.None)
|
||||
.Returns(Task.FromException<Elsa.Mediator.Models.Unit>(new InvalidOperationException("Queue unavailable.")));
|
||||
_store.DeleteAsync(failedItem.Id, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromException(new InvalidOperationException("Delete failed.")));
|
||||
|
||||
await _processor.ProcessAsync();
|
||||
|
||||
await _commandSender.Received(1).SendAsync(
|
||||
nextItem.WorkflowDefinitionCommand!,
|
||||
CommandStrategy.Background,
|
||||
Arg.Any<IDictionary<object, object>>(),
|
||||
CancellationToken.None);
|
||||
await _store.Received(1).SaveAsync(Arg.Is<WorkflowDispatchOutboxItem>(x => x.Id == failedItem.Id && x.DeliveryAttempts == _options.MaxOutboxDeliveryAttempts), Arg.Any<CancellationToken>());
|
||||
await _store.Received(1).DeleteAsync(nextItem.Id, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
private static WorkflowDispatchOutboxItem CreateItem(DateTimeOffset? createdAt = null, string id = "outbox-1")
|
||||
{
|
||||
return new()
|
||||
{
|
||||
Id = id,
|
||||
OwnerWorkflowInstanceId = "parent-1",
|
||||
Kind = WorkflowDispatchOutboxItemKind.WorkflowDefinition,
|
||||
WorkflowDefinitionCommand = new DispatchWorkflowDefinitionCommand("definition-version-1")
|
||||
{
|
||||
InstanceId = $"child-{id}"
|
||||
},
|
||||
CreatedAt = createdAt ?? DateTimeOffset.UtcNow
|
||||
};
|
||||
}
|
||||
|
||||
private static WorkflowInstance CreateOwnerWorkflowInstance(bool includeOutboxMarker, ICollection<string>? itemIds = null)
|
||||
{
|
||||
var workflowState = new WorkflowState
|
||||
{
|
||||
Id = "parent-1",
|
||||
DefinitionId = "parent-definition",
|
||||
DefinitionVersionId = "parent-definition-version",
|
||||
Properties = includeOutboxMarker
|
||||
? new Dictionary<string, object>
|
||||
{
|
||||
[WorkflowDispatchOutboxStateExtensions.PropertyKey] = new WorkflowDispatchOutboxState
|
||||
{
|
||||
ItemIds = itemIds ?? ["outbox-1"]
|
||||
}
|
||||
}
|
||||
: new Dictionary<string, object>()
|
||||
};
|
||||
|
||||
return new()
|
||||
{
|
||||
Id = workflowState.Id,
|
||||
DefinitionId = workflowState.DefinitionId,
|
||||
DefinitionVersionId = workflowState.DefinitionVersionId,
|
||||
WorkflowState = workflowState
|
||||
};
|
||||
}
|
||||
|
||||
private class ContendedDistributedSynchronizationProvider : IDistributedLockProvider
|
||||
{
|
||||
public IDistributedLock CreateLock(string name) => new ContendedDistributedLock(name);
|
||||
}
|
||||
|
||||
private class TimeoutDistributedSynchronizationProvider : IDistributedLockProvider
|
||||
{
|
||||
public IDistributedLock CreateLock(string name) => new TimeoutDistributedLock(name);
|
||||
}
|
||||
|
||||
private class RecordingDistributedSynchronizationProvider : IDistributedLockProvider
|
||||
{
|
||||
private readonly AsyncLocal<Stack<string>?> _heldLockNames = new();
|
||||
|
||||
public string? CurrentLockName => _heldLockNames.Value is { Count: > 0 } heldLockNames ? heldLockNames.Peek() : null;
|
||||
|
||||
public IDistributedLock CreateLock(string name) => new RecordingDistributedLock(this, name);
|
||||
|
||||
private IDistributedSynchronizationHandle Acquire(string name)
|
||||
{
|
||||
var heldLockNames = _heldLockNames.Value ??= new Stack<string>();
|
||||
heldLockNames.Push(name);
|
||||
return new RecordingDistributedSynchronizationHandle(this, name);
|
||||
}
|
||||
|
||||
private void Release(string name)
|
||||
{
|
||||
var heldLockNames = _heldLockNames.Value ?? throw new InvalidOperationException("No distributed locks are held.");
|
||||
var currentLockName = heldLockNames.Pop();
|
||||
|
||||
if (currentLockName != name)
|
||||
throw new InvalidOperationException($"Expected to release distributed lock '{currentLockName}', but '{name}' was released.");
|
||||
|
||||
if (heldLockNames.Count == 0)
|
||||
_heldLockNames.Value = null;
|
||||
}
|
||||
|
||||
private class RecordingDistributedLock(RecordingDistributedSynchronizationProvider provider, string name) : IDistributedLock
|
||||
{
|
||||
public string Name => name;
|
||||
|
||||
public IDistributedSynchronizationHandle? TryAcquire(TimeSpan timeout = default, CancellationToken cancellationToken = default) => provider.Acquire(name);
|
||||
|
||||
public IDistributedSynchronizationHandle Acquire(TimeSpan? timeout = null, CancellationToken cancellationToken = default) => provider.Acquire(name);
|
||||
|
||||
public ValueTask<IDistributedSynchronizationHandle?> TryAcquireAsync(TimeSpan timeout = default, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return new ValueTask<IDistributedSynchronizationHandle?>(provider.Acquire(name));
|
||||
}
|
||||
|
||||
public ValueTask<IDistributedSynchronizationHandle> AcquireAsync(TimeSpan? timeout = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return new ValueTask<IDistributedSynchronizationHandle>(provider.Acquire(name));
|
||||
}
|
||||
}
|
||||
|
||||
private class RecordingDistributedSynchronizationHandle(RecordingDistributedSynchronizationProvider provider, string name) : IDistributedSynchronizationHandle
|
||||
{
|
||||
private int _disposed;
|
||||
|
||||
public CancellationToken HandleLostToken => CancellationToken.None;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) == 0)
|
||||
provider.Release(name);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class ContendedDistributedLock(string name) : IDistributedLock
|
||||
{
|
||||
public string Name => name;
|
||||
|
||||
public IDistributedSynchronizationHandle? TryAcquire(TimeSpan timeout = default, CancellationToken cancellationToken = default) => null;
|
||||
|
||||
public IDistributedSynchronizationHandle Acquire(TimeSpan? timeout = null, CancellationToken cancellationToken = default) => throw new TimeoutException();
|
||||
|
||||
public virtual ValueTask<IDistributedSynchronizationHandle?> TryAcquireAsync(TimeSpan timeout = default, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return new ValueTask<IDistributedSynchronizationHandle?>((IDistributedSynchronizationHandle?)null);
|
||||
}
|
||||
|
||||
public ValueTask<IDistributedSynchronizationHandle> AcquireAsync(TimeSpan? timeout = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return ValueTask.FromException<IDistributedSynchronizationHandle>(new TimeoutException());
|
||||
}
|
||||
}
|
||||
|
||||
private class TimeoutDistributedLock(string name) : ContendedDistributedLock(name)
|
||||
{
|
||||
public override ValueTask<IDistributedSynchronizationHandle?> TryAcquireAsync(TimeSpan timeout = default, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return ValueTask.FromException<IDistributedSynchronizationHandle?>(new TimeoutException());
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue