TerraFX audio/video "pipeline" design notes

audio.cs

// N.B. these are "media" because they're intended to be pattern matched
// against for both audio and video
// e.g.:
// - `if (container is MatroskaContainer matroska)`
// - `if (codec is Vp8Codec vp8)`
// - `if (container is MpegContainer mpeg)`
// - `if (codec is Mp3Codec mp3)`

// format of audio/video data
interface IMediaCodec
{
    // user-facing name (e.g. 'Opus')
    string Name { get; }
    
    // implementors may have more information
    // e.g. sample rate for non-variable sample rate codecs
}

// container of audio/video data
interface IMediaContainer
{
    // user-facing name (e.g. 'OGG' or 'MPEG')
    string Name { get; }
    
    // implementors may have more information
    // e.g. number of streams for multi-stream files
}

// a raw frame of "media" e.g. a frame from a video file, or packet from an
// audio stream.
class MediaFrame : IDisposable
{
    // codec is here (instead of IAudioDecoder) as each frame may have
    // different codec settings
    public IMediaCodec Codec { get; }
    public Memory<byte> Data { get; }
    
    private readonly IMemoryOwner<byte> _data;
    private readonly int _length;
}

// reads audio from a device or file
interface IAudioRecordingDevice
{
    PipeReader Output { get; }
    IMediaContainer OutputContainer { get; }
    
    ValueTask RunAsync(CancellationToken cancelToken = default);
    
    // e.g.
    // var wavFile = new FileAudioRecordingDevice("path/to/file.wav");
}

// decodes media frames from a given container format
interface IAudioDecoder
{
    ChannelReader<MediaFrame> Output { get; }
    
    ValueTask RunAsync(CancellationToken cancelToken = default);
    
    // e.g.
    // var wavDecoder = new WaveAudioDecoder(wavFile.Output,
    //     wavFile.OutputContainer as WaveMediaContainer);
}

// transcodes audio frames from a given codec to another codec
interface IAudioTranscoder
{
    ChannelReader<MediaFrame> Output { get; }
    
    ValueTask RunAsync(CancellationToken cancelToken = default);
    
    // e.g.
    // var opusTranscoder = new OpusAudioTranscoder(wavFile.Output);
}

// encodes media frames to a given container format
interface IAudioEncoder
{
    PipeReader Output { get; }
    
    ValueTask RunAsync(CancellationToken cancelToken = default);
    
    // e.g.
    // var webmEncoder = new WebmAudioEncoder(opusTranscoder.Output);
}

// writes audio to a device or file
interface IAudioPlaybackDevice
{
    PipeWriter Input { get; }
    
    ValueTask RunAsync(CancellationToken cancelToken = default);
    
    // e.g.
    // var webmFile = new FileAudioPlaybackDevice("path/to/file.webm");
    // await webmEncoder.CopyToAsync(webmFile.Input);
}

// Example usage code:
// N.B. the user would likely be using factory methods to obtain these (e.g.
// OpusAudioTranscoderFactory.CreateTranscoder(input, codec)), as that will
// allow for simpler implementations (e.g. two different Opus transcoders for
// float vs ushort input)

var wavFile = new FileAudioRecordingDevice("path/to/file.wav");
Debug.Assert(wavFile.OutputConainer is WaveAudioContainer,
    "Input was not a wave file");

var wavDecoder = new WaveAudioDecoder(wavFile.Output,
    wavFile.OutputContainer as WaveMediaContainer);

var resampler = new PcmResamplingAudioTranscoder(wavDecoder.Output,
    new PcmAudioCodec
    {
        SampleRate = 48000
    });

var opusTranscoder = new OpusAudioTranscoder(resampler.Output);
var webmEncoder = new WebmAudioEncoder(opusTranscoder.Output);
var webmFile = new FileAudioPlaybackDevice("path/to/file.webm");

await Task.WhenAll(
    webmFile.RunAsync().AsTask(),
    webmEncoder.CopyToAsync(webmFile.Input),
    opusTranscoder.RunAsync().AsTask(),
    wavDecoder.RunAsync().AsTask(),
    wavFile.RunAsync().AsTask()
);

For lines 110-113, the codec could potentially be identified using wavDecoder.Output.TryPeek(out var frame).

Furthermore, MediaFrame could potentially be made virtual/abstract so that codecs can put more "stateful" information there, e.g. what stream a frame is from in multi-stream containers. (like Matroska)

It may be necessary to make the OutputContainer property on IAudioRecordingDevice an async getter (e.g. ValueTask<IMediaContainer> IdentifyOutputContainerAsync(CancellationToken cancelToken = default);) as we might not be able to identify the container synchronously due to requiring I/O