feat(pipeline): add IsoPipeline with lifecycle and restart supervisor
Some checks failed
CI / build-and-test (push) Failing after 29s

This commit is contained in:
Zac Gaetano 2026-05-07 15:26:54 +00:00
parent e318514202
commit 49b6dfb9ed
4 changed files with 333 additions and 11 deletions

View file

@ -0,0 +1,198 @@
using System.Threading.Channels;
using Microsoft.Extensions.Logging;
using TeamsISO.Engine.Domain;
using TeamsISO.Engine.Interop;
namespace TeamsISO.Engine.Pipeline;
/// <summary>
/// Per-ISO unit. Owns one capture loop, one frame processor, one send loop, and the
/// supervisor that restarts the inner pipeline with exponential backoff on failure.
/// </summary>
public sealed class IsoPipeline : IAsyncDisposable
{
private readonly Func<CancellationToken, Task> _runInner;
private readonly ExponentialBackoff _backoff;
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
private readonly ILogger<IsoPipeline> _logger;
private CancellationTokenSource? _cts;
private Task? _supervisorTask;
private int _consecutiveFailures;
public Guid ParticipantId { get; }
public IsoState State { get; private set; } = IsoState.Idle;
public int ConsecutiveFailures => _consecutiveFailures;
/// <summary>
/// Test ctor. The caller supplies the inner runner directly so failures and lifetimes
/// can be controlled from a unit test.
/// </summary>
internal IsoPipeline(
Guid participantId,
Func<CancellationToken, Task> runInner,
ExponentialBackoff backoff,
Func<TimeSpan, CancellationToken, Task> delay,
ILoggerFactory loggerFactory)
{
ParticipantId = participantId;
_runInner = runInner;
_backoff = backoff;
_delay = delay;
_logger = loggerFactory.CreateLogger<IsoPipeline>();
}
/// <summary>
/// Production ctor. Builds the inner runner from the engine dependencies.
/// </summary>
public IsoPipeline(
IsoPipelineConfig config,
INdiInterop interop,
IFrameScaler scaler,
IFrameClock frameClock,
ExponentialBackoff backoff,
Func<TimeSpan, CancellationToken, Task> delay,
ILoggerFactory loggerFactory)
: this(config.ParticipantId,
ct => RunInnerPipelineAsync(config, interop, scaler, frameClock, loggerFactory, ct),
backoff,
delay,
loggerFactory) { }
/// <summary>Starts the supervisor. Returns immediately; pipeline runs in the background.</summary>
public Task StartAsync()
{
if (_supervisorTask is not null)
throw new InvalidOperationException("Pipeline already started.");
_cts = new CancellationTokenSource();
State = IsoState.Receiving;
_supervisorTask = SupervisorLoopAsync(_cts.Token);
return Task.CompletedTask;
}
/// <summary>Stops the pipeline and awaits supervisor completion.</summary>
public async Task StopAsync()
{
if (_cts is null) return;
_cts.Cancel();
if (_supervisorTask is not null)
{
try { await _supervisorTask; }
catch (OperationCanceledException) { /* expected */ }
}
State = IsoState.Idle;
_cts.Dispose();
_cts = null;
_supervisorTask = null;
}
public async ValueTask DisposeAsync()
{
await StopAsync();
}
private async Task SupervisorLoopAsync(CancellationToken ct)
{
_consecutiveFailures = 0;
while (!ct.IsCancellationRequested)
{
try
{
await _runInner(ct);
// Inner exited normally (typically only on cancellation) — leave the loop.
break;
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_consecutiveFailures++;
_logger.LogWarning(ex,
"Pipeline {ParticipantId} failed (consecutive failure #{N}).",
ParticipantId, _consecutiveFailures);
if (_backoff.ShouldGiveUp(_consecutiveFailures))
{
State = IsoState.Error;
_logger.LogError("Pipeline {ParticipantId} giving up after {N} consecutive failures.",
ParticipantId, _consecutiveFailures);
return;
}
var wait = _backoff.NextDelay(_consecutiveFailures);
_logger.LogInformation("Pipeline {ParticipantId} retrying in {Delay}.", ParticipantId, wait);
try
{
await _delay(wait, ct);
}
catch (OperationCanceledException)
{
break;
}
State = IsoState.Receiving;
}
}
}
/// <summary>
/// Default inner pipeline: spins up receiver → processor → sender on bounded channels
/// and awaits all three. Throws if any of them throws.
/// </summary>
private static async Task RunInnerPipelineAsync(
IsoPipelineConfig config,
INdiInterop interop,
IFrameScaler scaler,
IFrameClock frameClock,
ILoggerFactory loggerFactory,
CancellationToken ct)
{
var rawChannel = Channel.CreateBounded<RawFrame>(new BoundedChannelOptions(config.RawChannelCapacity)
{
FullMode = BoundedChannelFullMode.DropOldest,
SingleReader = true,
SingleWriter = true,
});
var processedChannel = Channel.CreateBounded<ProcessedFrame>(new BoundedChannelOptions(config.ProcessedChannelCapacity)
{
FullMode = BoundedChannelFullMode.DropOldest,
SingleReader = true,
SingleWriter = true,
});
using var receiver = new NdiReceiver(
interop, config.SourceName, rawChannel.Writer, loggerFactory.CreateLogger<NdiReceiver>());
using var sender = new NdiSender(
interop, config.OutputName, processedChannel.Reader, loggerFactory.CreateLogger<NdiSender>());
var processor = new FrameProcessor(
config.Settings, scaler, new SolidFrameRenderer(),
frameClock, rawChannel.Reader, processedChannel.Writer,
config.SlateThreshold, loggerFactory.CreateLogger<FrameProcessor>());
var receiverTask = receiver.RunAsync(ct);
var senderTask = sender.RunAsync(ct);
var processorTask = ProcessorLoopAsync(processor, frameClock, ct);
try
{
await Task.WhenAll(receiverTask, senderTask, processorTask);
}
finally
{
rawChannel.Writer.TryComplete();
processedChannel.Writer.TryComplete();
}
}
private static async Task ProcessorLoopAsync(FrameProcessor processor, IFrameClock clock, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
var advanced = await clock.WaitForNextTickAsync(ct);
if (!advanced) break;
await processor.ProcessOnceAsync(ct);
}
}
}

View file

@ -0,0 +1,23 @@
using TeamsISO.Engine.Domain;
namespace TeamsISO.Engine.Pipeline;
/// <summary>
/// Per-pipeline configuration — identifies the participant, the source it captures,
/// the output it emits, and the global processing settings to apply.
/// </summary>
public sealed record IsoPipelineConfig(
Guid ParticipantId,
string SourceName,
string OutputName,
FrameProcessingSettings Settings)
{
/// <summary>Default no-signal threshold per spec §4.</summary>
public TimeSpan SlateThreshold { get; init; } = TimeSpan.FromSeconds(2.5);
/// <summary>Bounded raw-frame channel capacity (drop-oldest backpressure).</summary>
public int RawChannelCapacity { get; init; } = 4;
/// <summary>Bounded processed-frame channel capacity.</summary>
public int ProcessedChannelCapacity { get; init; } = 2;
}

View file

@ -1,17 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Serilog" Version="4.0.0" />
<PackageReference Include="Serilog.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="System.Reactive" Version="6.0.0" />
</ItemGroup>
</Project>
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="TeamsISO.Engine.Tests" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,99 @@
using Microsoft.Extensions.Logging.Abstractions;
using TeamsISO.Engine.Domain;
using TeamsISO.Engine.Pipeline;
namespace TeamsISO.Engine.Tests.Pipeline;
public class IsoPipelineTests
{
private static ExponentialBackoff FastBackoff() =>
new(maxAttempts: 5, initial: TimeSpan.FromMilliseconds(1), cap: TimeSpan.FromMilliseconds(10));
private static Func<TimeSpan, CancellationToken, Task> NoDelay() =>
(_, _) => Task.CompletedTask;
[Fact]
public async Task StartAsync_TransitionsIdleToReceiving()
{
var participant = Guid.NewGuid();
var blocker = new TaskCompletionSource();
var pipeline = new IsoPipeline(
participant,
ct => blocker.Task.WaitAsync(ct),
FastBackoff(),
NoDelay(),
NullLoggerFactory.Instance);
await pipeline.StartAsync();
pipeline.State.Should().Be(IsoState.Receiving);
blocker.SetResult();
await pipeline.StopAsync();
}
[Fact]
public async Task StopAsync_TransitionsBackToIdle()
{
var blocker = new TaskCompletionSource();
var pipeline = new IsoPipeline(
Guid.NewGuid(),
ct => blocker.Task.WaitAsync(ct),
FastBackoff(),
NoDelay(),
NullLoggerFactory.Instance);
await pipeline.StartAsync();
await pipeline.StopAsync();
pipeline.State.Should().Be(IsoState.Idle);
}
[Fact]
public async Task Supervisor_FailsOnce_ThenRecovers_RemainsReceiving()
{
int callCount = 0;
var done = new TaskCompletionSource();
var pipeline = new IsoPipeline(
Guid.NewGuid(),
ct =>
{
callCount++;
if (callCount == 1) throw new InvalidOperationException("boom");
return done.Task.WaitAsync(ct);
},
FastBackoff(),
NoDelay(),
NullLoggerFactory.Instance);
await pipeline.StartAsync();
await Task.Delay(50); // let the supervisor cycle through the failure
callCount.Should().BeGreaterOrEqualTo(2);
pipeline.ConsecutiveFailures.Should().Be(1);
pipeline.State.Should().Be(IsoState.Receiving);
done.SetResult();
await pipeline.StopAsync();
}
[Fact]
public async Task Supervisor_FailsRepeatedly_TransitionsToError()
{
var pipeline = new IsoPipeline(
Guid.NewGuid(),
_ => throw new InvalidOperationException("permanent"),
FastBackoff(),
NoDelay(),
NullLoggerFactory.Instance);
await pipeline.StartAsync();
// Wait for supervisor to give up (max 5 attempts × ~1ms delay)
var deadline = DateTime.UtcNow.AddSeconds(2);
while (pipeline.State != IsoState.Error && DateTime.UtcNow < deadline)
await Task.Delay(20);
pipeline.State.Should().Be(IsoState.Error);
pipeline.ConsecutiveFailures.Should().Be(6); // 5 retries allowed, give up on 6th
}
}