feat(pipeline): add IsoPipeline with lifecycle and restart supervisor
Some checks failed
CI / build-and-test (push) Failing after 29s
Some checks failed
CI / build-and-test (push) Failing after 29s
This commit is contained in:
parent
e318514202
commit
49b6dfb9ed
4 changed files with 333 additions and 11 deletions
198
src/TeamsISO.Engine/Pipeline/IsoPipeline.cs
Normal file
198
src/TeamsISO.Engine/Pipeline/IsoPipeline.cs
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
using System.Threading.Channels;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using TeamsISO.Engine.Domain;
|
||||
using TeamsISO.Engine.Interop;
|
||||
|
||||
namespace TeamsISO.Engine.Pipeline;
|
||||
|
||||
/// <summary>
|
||||
/// Per-ISO unit. Owns one capture loop, one frame processor, one send loop, and the
|
||||
/// supervisor that restarts the inner pipeline with exponential backoff on failure.
|
||||
/// </summary>
|
||||
public sealed class IsoPipeline : IAsyncDisposable
|
||||
{
|
||||
private readonly Func<CancellationToken, Task> _runInner;
|
||||
private readonly ExponentialBackoff _backoff;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
|
||||
private readonly ILogger<IsoPipeline> _logger;
|
||||
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _supervisorTask;
|
||||
private int _consecutiveFailures;
|
||||
|
||||
public Guid ParticipantId { get; }
|
||||
public IsoState State { get; private set; } = IsoState.Idle;
|
||||
public int ConsecutiveFailures => _consecutiveFailures;
|
||||
|
||||
/// <summary>
|
||||
/// Test ctor. The caller supplies the inner runner directly so failures and lifetimes
|
||||
/// can be controlled from a unit test.
|
||||
/// </summary>
|
||||
internal IsoPipeline(
|
||||
Guid participantId,
|
||||
Func<CancellationToken, Task> runInner,
|
||||
ExponentialBackoff backoff,
|
||||
Func<TimeSpan, CancellationToken, Task> delay,
|
||||
ILoggerFactory loggerFactory)
|
||||
{
|
||||
ParticipantId = participantId;
|
||||
_runInner = runInner;
|
||||
_backoff = backoff;
|
||||
_delay = delay;
|
||||
_logger = loggerFactory.CreateLogger<IsoPipeline>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Production ctor. Builds the inner runner from the engine dependencies.
|
||||
/// </summary>
|
||||
public IsoPipeline(
|
||||
IsoPipelineConfig config,
|
||||
INdiInterop interop,
|
||||
IFrameScaler scaler,
|
||||
IFrameClock frameClock,
|
||||
ExponentialBackoff backoff,
|
||||
Func<TimeSpan, CancellationToken, Task> delay,
|
||||
ILoggerFactory loggerFactory)
|
||||
: this(config.ParticipantId,
|
||||
ct => RunInnerPipelineAsync(config, interop, scaler, frameClock, loggerFactory, ct),
|
||||
backoff,
|
||||
delay,
|
||||
loggerFactory) { }
|
||||
|
||||
/// <summary>Starts the supervisor. Returns immediately; pipeline runs in the background.</summary>
|
||||
public Task StartAsync()
|
||||
{
|
||||
if (_supervisorTask is not null)
|
||||
throw new InvalidOperationException("Pipeline already started.");
|
||||
_cts = new CancellationTokenSource();
|
||||
State = IsoState.Receiving;
|
||||
_supervisorTask = SupervisorLoopAsync(_cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Stops the pipeline and awaits supervisor completion.</summary>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
if (_cts is null) return;
|
||||
_cts.Cancel();
|
||||
if (_supervisorTask is not null)
|
||||
{
|
||||
try { await _supervisorTask; }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
}
|
||||
State = IsoState.Idle;
|
||||
_cts.Dispose();
|
||||
_cts = null;
|
||||
_supervisorTask = null;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await StopAsync();
|
||||
}
|
||||
|
||||
private async Task SupervisorLoopAsync(CancellationToken ct)
|
||||
{
|
||||
_consecutiveFailures = 0;
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _runInner(ct);
|
||||
// Inner exited normally (typically only on cancellation) — leave the loop.
|
||||
break;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_consecutiveFailures++;
|
||||
_logger.LogWarning(ex,
|
||||
"Pipeline {ParticipantId} failed (consecutive failure #{N}).",
|
||||
ParticipantId, _consecutiveFailures);
|
||||
|
||||
if (_backoff.ShouldGiveUp(_consecutiveFailures))
|
||||
{
|
||||
State = IsoState.Error;
|
||||
_logger.LogError("Pipeline {ParticipantId} giving up after {N} consecutive failures.",
|
||||
ParticipantId, _consecutiveFailures);
|
||||
return;
|
||||
}
|
||||
|
||||
var wait = _backoff.NextDelay(_consecutiveFailures);
|
||||
_logger.LogInformation("Pipeline {ParticipantId} retrying in {Delay}.", ParticipantId, wait);
|
||||
try
|
||||
{
|
||||
await _delay(wait, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
State = IsoState.Receiving;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default inner pipeline: spins up receiver → processor → sender on bounded channels
|
||||
/// and awaits all three. Throws if any of them throws.
|
||||
/// </summary>
|
||||
private static async Task RunInnerPipelineAsync(
|
||||
IsoPipelineConfig config,
|
||||
INdiInterop interop,
|
||||
IFrameScaler scaler,
|
||||
IFrameClock frameClock,
|
||||
ILoggerFactory loggerFactory,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var rawChannel = Channel.CreateBounded<RawFrame>(new BoundedChannelOptions(config.RawChannelCapacity)
|
||||
{
|
||||
FullMode = BoundedChannelFullMode.DropOldest,
|
||||
SingleReader = true,
|
||||
SingleWriter = true,
|
||||
});
|
||||
var processedChannel = Channel.CreateBounded<ProcessedFrame>(new BoundedChannelOptions(config.ProcessedChannelCapacity)
|
||||
{
|
||||
FullMode = BoundedChannelFullMode.DropOldest,
|
||||
SingleReader = true,
|
||||
SingleWriter = true,
|
||||
});
|
||||
|
||||
using var receiver = new NdiReceiver(
|
||||
interop, config.SourceName, rawChannel.Writer, loggerFactory.CreateLogger<NdiReceiver>());
|
||||
using var sender = new NdiSender(
|
||||
interop, config.OutputName, processedChannel.Reader, loggerFactory.CreateLogger<NdiSender>());
|
||||
|
||||
var processor = new FrameProcessor(
|
||||
config.Settings, scaler, new SolidFrameRenderer(),
|
||||
frameClock, rawChannel.Reader, processedChannel.Writer,
|
||||
config.SlateThreshold, loggerFactory.CreateLogger<FrameProcessor>());
|
||||
|
||||
var receiverTask = receiver.RunAsync(ct);
|
||||
var senderTask = sender.RunAsync(ct);
|
||||
var processorTask = ProcessorLoopAsync(processor, frameClock, ct);
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(receiverTask, senderTask, processorTask);
|
||||
}
|
||||
finally
|
||||
{
|
||||
rawChannel.Writer.TryComplete();
|
||||
processedChannel.Writer.TryComplete();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ProcessorLoopAsync(FrameProcessor processor, IFrameClock clock, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var advanced = await clock.WaitForNextTickAsync(ct);
|
||||
if (!advanced) break;
|
||||
await processor.ProcessOnceAsync(ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
23
src/TeamsISO.Engine/Pipeline/IsoPipelineConfig.cs
Normal file
23
src/TeamsISO.Engine/Pipeline/IsoPipelineConfig.cs
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
using TeamsISO.Engine.Domain;
|
||||
|
||||
namespace TeamsISO.Engine.Pipeline;
|
||||
|
||||
/// <summary>
|
||||
/// Per-pipeline configuration — identifies the participant, the source it captures,
|
||||
/// the output it emits, and the global processing settings to apply.
|
||||
/// </summary>
|
||||
public sealed record IsoPipelineConfig(
|
||||
Guid ParticipantId,
|
||||
string SourceName,
|
||||
string OutputName,
|
||||
FrameProcessingSettings Settings)
|
||||
{
|
||||
/// <summary>Default no-signal threshold per spec §4.</summary>
|
||||
public TimeSpan SlateThreshold { get; init; } = TimeSpan.FromSeconds(2.5);
|
||||
|
||||
/// <summary>Bounded raw-frame channel capacity (drop-oldest backpressure).</summary>
|
||||
public int RawChannelCapacity { get; init; } = 4;
|
||||
|
||||
/// <summary>Bounded processed-frame channel capacity.</summary>
|
||||
public int ProcessedChannelCapacity { get; init; } = 2;
|
||||
}
|
||||
|
|
@ -1,17 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Serilog" Version="4.0.0" />
|
||||
<PackageReference Include="Serilog.Extensions.Logging" Version="8.0.0" />
|
||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
|
||||
<PackageReference Include="System.Reactive" Version="6.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="TeamsISO.Engine.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
|||
99
src/tests/TeamsISO.Engine.Tests/Pipeline/IsoPipelineTests.cs
Normal file
99
src/tests/TeamsISO.Engine.Tests/Pipeline/IsoPipelineTests.cs
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using TeamsISO.Engine.Domain;
|
||||
using TeamsISO.Engine.Pipeline;
|
||||
|
||||
namespace TeamsISO.Engine.Tests.Pipeline;
|
||||
|
||||
public class IsoPipelineTests
|
||||
{
|
||||
private static ExponentialBackoff FastBackoff() =>
|
||||
new(maxAttempts: 5, initial: TimeSpan.FromMilliseconds(1), cap: TimeSpan.FromMilliseconds(10));
|
||||
|
||||
private static Func<TimeSpan, CancellationToken, Task> NoDelay() =>
|
||||
(_, _) => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task StartAsync_TransitionsIdleToReceiving()
|
||||
{
|
||||
var participant = Guid.NewGuid();
|
||||
var blocker = new TaskCompletionSource();
|
||||
var pipeline = new IsoPipeline(
|
||||
participant,
|
||||
ct => blocker.Task.WaitAsync(ct),
|
||||
FastBackoff(),
|
||||
NoDelay(),
|
||||
NullLoggerFactory.Instance);
|
||||
|
||||
await pipeline.StartAsync();
|
||||
|
||||
pipeline.State.Should().Be(IsoState.Receiving);
|
||||
|
||||
blocker.SetResult();
|
||||
await pipeline.StopAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StopAsync_TransitionsBackToIdle()
|
||||
{
|
||||
var blocker = new TaskCompletionSource();
|
||||
var pipeline = new IsoPipeline(
|
||||
Guid.NewGuid(),
|
||||
ct => blocker.Task.WaitAsync(ct),
|
||||
FastBackoff(),
|
||||
NoDelay(),
|
||||
NullLoggerFactory.Instance);
|
||||
|
||||
await pipeline.StartAsync();
|
||||
await pipeline.StopAsync();
|
||||
|
||||
pipeline.State.Should().Be(IsoState.Idle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Supervisor_FailsOnce_ThenRecovers_RemainsReceiving()
|
||||
{
|
||||
int callCount = 0;
|
||||
var done = new TaskCompletionSource();
|
||||
var pipeline = new IsoPipeline(
|
||||
Guid.NewGuid(),
|
||||
ct =>
|
||||
{
|
||||
callCount++;
|
||||
if (callCount == 1) throw new InvalidOperationException("boom");
|
||||
return done.Task.WaitAsync(ct);
|
||||
},
|
||||
FastBackoff(),
|
||||
NoDelay(),
|
||||
NullLoggerFactory.Instance);
|
||||
|
||||
await pipeline.StartAsync();
|
||||
await Task.Delay(50); // let the supervisor cycle through the failure
|
||||
callCount.Should().BeGreaterOrEqualTo(2);
|
||||
pipeline.ConsecutiveFailures.Should().Be(1);
|
||||
pipeline.State.Should().Be(IsoState.Receiving);
|
||||
|
||||
done.SetResult();
|
||||
await pipeline.StopAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Supervisor_FailsRepeatedly_TransitionsToError()
|
||||
{
|
||||
var pipeline = new IsoPipeline(
|
||||
Guid.NewGuid(),
|
||||
_ => throw new InvalidOperationException("permanent"),
|
||||
FastBackoff(),
|
||||
NoDelay(),
|
||||
NullLoggerFactory.Instance);
|
||||
|
||||
await pipeline.StartAsync();
|
||||
|
||||
// Wait for supervisor to give up (max 5 attempts × ~1ms delay)
|
||||
var deadline = DateTime.UtcNow.AddSeconds(2);
|
||||
while (pipeline.State != IsoState.Error && DateTime.UtcNow < deadline)
|
||||
await Task.Delay(20);
|
||||
|
||||
pipeline.State.Should().Be(IsoState.Error);
|
||||
pipeline.ConsecutiveFailures.Should().Be(6); // 5 retries allowed, give up on 6th
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue