diff --git a/Beutl.sln b/Beutl.sln index cd6fbac31..c7cccd5f6 100644 --- a/Beutl.sln +++ b/Beutl.sln @@ -109,6 +109,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Beutl.Extensions.MediaFound EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Beutl.PackageTools.UI", "src\Beutl.PackageTools.UI\Beutl.PackageTools.UI.csproj", "{D8A8061C-CE79-4DF7-B9E8-2002BAD47DD8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Beutl.Extensions.AVFoundation", "src\Beutl.Extensions.AVFoundation\Beutl.Extensions.AVFoundation.csproj", "{8B040DCA-6C9C-4009-8FE5-8F764D80907B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -371,6 +373,14 @@ Global {D8A8061C-CE79-4DF7-B9E8-2002BAD47DD8}.Release|Any CPU.Build.0 = Release|Any CPU {D8A8061C-CE79-4DF7-B9E8-2002BAD47DD8}.Release|x64.ActiveCfg = Release|Any CPU {D8A8061C-CE79-4DF7-B9E8-2002BAD47DD8}.Release|x64.Build.0 = Release|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Debug|x64.ActiveCfg = Debug|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Debug|x64.Build.0 = Debug|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Release|Any CPU.Build.0 = Release|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Release|x64.ActiveCfg = Release|Any CPU + {8B040DCA-6C9C-4009-8FE5-8F764D80907B}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Directory.Packages.props b/Directory.Packages.props index dba524fa5..75b1ca314 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -27,6 +27,7 @@ + @@ -80,4 +81,4 @@ - \ No newline at end of file + diff --git a/src/Beutl.Extensions.AVFoundation/AVFSampleUtilities.cs b/src/Beutl.Extensions.AVFoundation/AVFSampleUtilities.cs new file mode 100644 index 000000000..d679c5997 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/AVFSampleUtilities.cs @@ -0,0 +1,98 @@ +using System.Diagnostics; +using Beutl.Media; +using Beutl.Media.Pixel; +using MonoMac.CoreGraphics; +using MonoMac.CoreImage; +using MonoMac.CoreMedia; +using MonoMac.CoreVideo; +using MonoMac.Foundation; + +namespace Beutl.Extensions.AVFoundation; + +public class AVFSampleUtilities +{ + public static unsafe CVPixelBuffer? ConvertToCVPixelBuffer(Bitmap bitmap) + { + int width = bitmap.Width; + int height = bitmap.Height; + var pixelBuffer = new CVPixelBuffer(width, height, CVPixelFormatType.CV32BGRA, new CVPixelBufferAttributes + { + PixelFormatType = CVPixelFormatType.CV32BGRA, + Width = width, + Height = height + }); + + var r = pixelBuffer.Lock(CVOptionFlags.None); + if (r != CVReturn.Success) return null; + + Buffer.MemoryCopy( + (void*)bitmap.Data, (void*)pixelBuffer.GetBaseAddress(0), + bitmap.ByteCount, bitmap.ByteCount); + + pixelBuffer.Unlock(CVOptionFlags.None); + + return pixelBuffer; + } + + public static unsafe Bitmap? ConvertToBgra(CMSampleBuffer buffer) + { + using var imageBuffer = buffer.GetImageBuffer(); + if (imageBuffer is not CVPixelBuffer pixelBuffer) return null; + + var r = pixelBuffer.Lock(CVOptionFlags.None); + if (r != CVReturn.Success) return null; + + int width = pixelBuffer.Width; + int height = pixelBuffer.Height; + var bitmap = new Bitmap(width, height); + if (pixelBuffer.ColorSpace.Model == CGColorSpaceModel.RGB && pixelBuffer.BytesPerRow == width * 4) + { + Buffer.MemoryCopy( + (void*)pixelBuffer.GetBaseAddress(0), (void*)bitmap.Data, + bitmap.ByteCount, bitmap.ByteCount); + pixelBuffer.Unlock(CVOptionFlags.None); + Parallel.For(0, width * height, i => + { + // argb + // bgra + var o = bitmap.DataSpan[i]; + bitmap.DataSpan[i] = new Bgra8888(o.G, o.R, o.A, o.B); + }); + return bitmap; + } + + int bytesPerRow = width * height * 4; + using (CGColorSpace colorSpace = CGColorSpace.CreateDeviceRGB()) + using (var cgContext = new CGBitmapContext( + bitmap.Data, width, height, + 8, bytesPerRow, colorSpace, + CGBitmapFlags.ByteOrderDefault | CGBitmapFlags.PremultipliedFirst)) + using (var ciImage = CIImage.FromImageBuffer(imageBuffer)) + using (var ciContext = new CIContext(NSObjectFlag.Empty)) + // CreateCGImageで落ちる、例外なしに + using (var cgImage = ciContext.CreateCGImage( + ciImage, new CGRect(0, 0, width, height), (long)CIFormat.ARGB8, colorSpace)) + { + cgContext.DrawImage(new CGRect(0, 0, width, height), cgImage); + } + + pixelBuffer.Unlock(CVOptionFlags.None); + + return bitmap; + } + + public static int SampleCopyToBuffer(CMSampleBuffer buffer, nint buf, int copyBufferPos, + int copyBufferSize) + { + using var dataBuffer = buffer.GetDataBuffer(); + Debug.Assert((copyBufferPos + copyBufferSize) <= dataBuffer.DataLength); + dataBuffer.CopyDataBytes((uint)copyBufferPos, (uint)copyBufferSize, buf); + + return copyBufferSize; + } + + public static int SampleCopyToBuffer(CMSampleBuffer buffer, nint buf, int copyBufferSize) + { + return SampleCopyToBuffer(buffer, buf, 0, copyBufferSize); + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Beutl.Extensions.AVFoundation.csproj b/src/Beutl.Extensions.AVFoundation/Beutl.Extensions.AVFoundation.csproj new file mode 100644 index 000000000..6fa8fdb69 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Beutl.Extensions.AVFoundation.csproj @@ -0,0 +1,15 @@ + + + + false + + + + + + + + + + + diff --git a/src/Beutl.Extensions.AVFoundation/CMTimeUtilities.cs b/src/Beutl.Extensions.AVFoundation/CMTimeUtilities.cs new file mode 100644 index 000000000..5e0a391e2 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/CMTimeUtilities.cs @@ -0,0 +1,16 @@ +using MonoMac.CoreMedia; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +internal static class CMTimeUtilities +{ + public static int ConvertFrameFromTimeStamp(CMTime timestamp, double rate) + { + return (int)Math.Round(timestamp.Seconds * rate, MidpointRounding.AwayFromZero); + } + + public static CMTime ConvertTimeStampFromFrame(int frame, double rate) + { + return CMTime.FromSeconds(frame / rate, 1); + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFAudioSampleCache.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFAudioSampleCache.cs new file mode 100644 index 000000000..42d23e9e0 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFAudioSampleCache.cs @@ -0,0 +1,127 @@ +using Beutl.Collections; +using Beutl.Logging; +using Microsoft.Extensions.Logging; +using MonoMac.CoreMedia; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +public class AVFAudioSampleCache(AVFSampleCacheOptions options) +{ + private readonly ILogger _logger = Log.CreateLogger(); + + public const int AudioSampleWaringGapCount = 1000; + + private CircularBuffer _audioCircularBuffer = new(options.MaxAudioBufferSize); + private short _nBlockAlign; + + private readonly record struct AudioCache(int StartSampleNum, CMSampleBuffer Sample, int AudioSampleCount) + { + public bool CopyBuffer(ref int startSample, ref int copySampleLength, ref nint buffer, short nBlockAlign) + { + int querySampleEndPos = startSample + copySampleLength; + int cacheSampleEndPos = StartSampleNum + AudioSampleCount; + // キャッシュ内に startSample位置があるかどうか + if (StartSampleNum <= startSample && startSample < cacheSampleEndPos) + { + // 要求サイズがキャッシュを超えるかどうか + if (querySampleEndPos <= cacheSampleEndPos) + { + // キャッシュ内に収まる + int actualBufferPos = (startSample - StartSampleNum) * nBlockAlign; + int actualBufferSize = copySampleLength * nBlockAlign; + AVFSampleUtilities.SampleCopyToBuffer(Sample, buffer, actualBufferPos, actualBufferSize); + + startSample += copySampleLength; + copySampleLength = 0; + buffer += actualBufferSize; + + return true; + } + else + { + // 現在のキャッシュ内のデータをコピーする + int actualBufferPos = (startSample - StartSampleNum) * nBlockAlign; + int leftSampleCount = cacheSampleEndPos - startSample; + int actualleftBufferSize = leftSampleCount * nBlockAlign; + AVFSampleUtilities.SampleCopyToBuffer(Sample, buffer, actualBufferPos, actualleftBufferSize); + + startSample += leftSampleCount; + copySampleLength -= leftSampleCount; + buffer += actualleftBufferSize; + + return true; + } + } + + return false; + } + } + + public void Reset(short nBlockAlign) + { + _nBlockAlign = nBlockAlign; + CircularBuffer old = _audioCircularBuffer; + _audioCircularBuffer = new CircularBuffer(options.MaxAudioBufferSize); + foreach (AudioCache item in old) + { + item.Sample.Dispose(); + } + } + + public void Add(int startSample, CMSampleBuffer buffer) + { + int lastAudioSampleNum = LastAudioSampleNumber(); + if (lastAudioSampleNum != -1) + { + int actualAudioSampleNum = lastAudioSampleNum + _audioCircularBuffer.Back().AudioSampleCount; + if (Math.Abs(startSample - actualAudioSampleNum) > AudioSampleWaringGapCount) + { + _logger.LogWarning( + "sample laggin - lag: {lag} startSample: {startSample} lastAudioSampleNum: {lastAudioSampleNum}", + startSample - actualAudioSampleNum, + startSample, + actualAudioSampleNum); + } + + startSample = lastAudioSampleNum + _audioCircularBuffer.Back().AudioSampleCount; + } + + int audioSampleCount = buffer.NumSamples; + + if (_audioCircularBuffer.IsFull) + { + _audioCircularBuffer.Front().Sample.Dispose(); + _audioCircularBuffer.PopFront(); + } + + var audioCache = new AudioCache(startSample, buffer, audioSampleCount); + _audioCircularBuffer.PushBack(audioCache); + } + + public int LastAudioSampleNumber() + { + if (_audioCircularBuffer.Size > 0) + { + AudioCache prevAudioCache = _audioCircularBuffer.Back(); + return prevAudioCache.StartSampleNum; + } + + return -1; + } + + public bool SearchAudioSampleAndCopyBuffer(int startSample, int copySampleLength, nint buffer) + { + foreach (AudioCache audioCache in _audioCircularBuffer) + { + if (audioCache.CopyBuffer(ref startSample, ref copySampleLength, ref buffer, _nBlockAlign)) + { + if (copySampleLength == 0) + { + return true; + } + } + } + + return false; + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFAudioStreamReader.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFAudioStreamReader.cs new file mode 100644 index 000000000..94cb3f367 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFAudioStreamReader.cs @@ -0,0 +1,224 @@ +using System.Diagnostics.CodeAnalysis; +using Beutl.Logging; +using Beutl.Media.Decoding; +using Beutl.Media.Music; +using Beutl.Media.Music.Samples; +using Microsoft.Extensions.Logging; +using MonoMac.AudioToolbox; +using MonoMac.AVFoundation; +using MonoMac.CoreMedia; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +public class AVFAudioStreamReader : IDisposable +{ + private readonly ILogger _logger = Log.CreateLogger(); + private readonly AVAsset _asset; + private readonly MediaOptions _options; + private readonly AVFAudioSampleCache _sampleCache; + private readonly int _thresholdSampleCount; + + private readonly AVAssetTrack _audioTrack; + private AVAssetReader _assetAudioReader; + private AVAssetReaderTrackOutput _audioReaderOutput; + private CMTime _currentAudioTimestamp = CMTime.Zero; + private CMTime _firstGapTimestamp = CMTime.Zero; + + public AVFAudioStreamReader(AVAsset asset, MediaOptions options, AVFDecodingExtension extension) + { + _asset = asset; + _options = options; + _sampleCache = + new AVFAudioSampleCache( + new AVFSampleCacheOptions(MaxAudioBufferSize: extension.Settings?.MaxAudioBufferSize ?? 20)); + _thresholdSampleCount = extension.Settings?.ThresholdSampleCount ?? 30000; + + _sampleCache.Reset(4 * 2); + + _audioTrack = _asset.TracksWithMediaType(AVMediaType.Audio)[0]; + + _assetAudioReader = AVAssetReader.FromAsset(_asset, out var error); + if (error != null) throw new Exception(error.LocalizedDescription); + + _audioReaderOutput = new AVAssetReaderTrackOutput( + _audioTrack, + new AudioSettings() + { + Format = AudioFormatType.LinearPCM, + LinearPcmBitDepth = 32, + LinearPcmBigEndian = false, + LinearPcmFloat = true, + SampleRate = options.SampleRate, + NumberChannels = 2, + }.Dictionary); + _assetAudioReader.AddOutput(_audioReaderOutput); + + _assetAudioReader.StartReading(); + + var audioDesc = _audioTrack.FormatDescriptions[0]; + AudioInfo = new AudioStreamInfo( + audioDesc.AudioFormatType.ToString(), + Rational.FromDouble(_audioTrack.TotalSampleDataLength / _audioTrack.EstimatedDataRate * 8d), + _audioTrack.NaturalTimeScale, + audioDesc.AudioChannelLayout.Channels.Length); + + TestFirstReadSample(); + } + + ~AVFAudioStreamReader() + { + if (!IsDisposed) + { + DisposeCore(false); + } + } + + public bool IsDisposed { get; private set; } + + public AudioStreamInfo AudioInfo { get; } + + private void SeekAudio(CMTime timestamp) + { + // _audioTrack. + _sampleCache.Reset(4 * 2); + _assetAudioReader.Dispose(); + _audioReaderOutput.Dispose(); + + _assetAudioReader = AVAssetReader.FromAsset(_asset, out var error); + if (error != null) throw new Exception(error.LocalizedDescription); + _assetAudioReader.TimeRange = new CMTimeRange { Start = timestamp, Duration = CMTime.PositiveInfinity }; + + _audioReaderOutput = new AVAssetReaderTrackOutput( + _audioTrack, + new AudioSettings() + { + Format = AudioFormatType.LinearPCM, + LinearPcmBitDepth = 32, + LinearPcmBigEndian = false, + LinearPcmFloat = true, + SampleRate = _options.SampleRate, + NumberChannels = 2, + }.Dictionary); + _audioReaderOutput.AlwaysCopiesSampleData = false; + _assetAudioReader.AddOutput(_audioReaderOutput); + + _assetAudioReader.StartReading(); + } + + public bool ReadAudio(int start, int length, [NotNullWhen(true)] out IPcm? sound) + { + start = (int)((long)_options.SampleRate * start / AudioInfo.SampleRate); + length = (int)((long)_options.SampleRate * length / AudioInfo.SampleRate); + var buffer = new Pcm(_options.SampleRate, length); + bool hitCache = _sampleCache.SearchAudioSampleAndCopyBuffer(start, length, buffer.Data); + if (hitCache) + { + sound = buffer; + return true; + } + + int currentSample = _sampleCache.LastAudioSampleNumber(); + if (currentSample == -1) + { + currentSample = (int)_currentAudioTimestamp.Value; + } + + if (start < currentSample || (currentSample + _thresholdSampleCount) < start) + { + var destTimePosition = new CMTime(start, _options.SampleRate); + SeekAudio(destTimePosition); + } + + CMSampleBuffer? sample = ReadAudioSample(); + + while (sample != null) + { + try + { + int readSampleNum = _sampleCache.LastAudioSampleNumber(); + + if (start <= readSampleNum) + { + if (_sampleCache.SearchAudioSampleAndCopyBuffer(start, length, buffer.Data)) + { + sound = buffer; + return true; + } + } + + sample = ReadAudioSample(); + } + catch + { + break; + } + } + + sound = buffer; + return true; + } + + private CMSampleBuffer? ReadAudioSample() + { + var buffer = _audioReaderOutput.CopyNextSampleBuffer(); + if (!buffer.DataIsReady) + { + _logger.LogTrace("buffer.DataIsReady = false"); + + buffer = _audioReaderOutput.CopyNextSampleBuffer(); + if (!buffer.DataIsReady) + { + _logger.LogTrace("2 buffer.DataIsReady = false"); + return null; + } + + // return null; + } + + if (!buffer.IsValid) + { + _logger.LogTrace("buffer is invalid."); + return null; + } + + // success! + // add cache + var timestamp = buffer.PresentationTimeStamp; + timestamp -= _firstGapTimestamp; + int startSample = (int)timestamp.Value; + _sampleCache.Add(startSample, buffer); + _currentAudioTimestamp = timestamp; + + return buffer; + } + + private void TestFirstReadSample() + { + _ = ReadAudioSample() ?? throw new Exception("TestFirstReadSample() failed"); + _logger.LogInformation( + "TestFirstReadSample firstTimeStamp: {currentAudioTimeStamp}", + _currentAudioTimestamp); + CMTime firstAudioTimeStamp = _currentAudioTimestamp; + SeekAudio(CMTime.Zero); + _currentAudioTimestamp = CMTime.Zero; + + _firstGapTimestamp = firstAudioTimeStamp; + _logger.LogInformation("TestFirstReadSample - firstGapTimeStamp: {firstGapTimeStamp}", _firstGapTimestamp); + } + + private void DisposeCore(bool disposing) + { + _sampleCache.Reset(0); + _audioReaderOutput.Dispose(); + _assetAudioReader.Dispose(); + } + + public void Dispose() + { + if (IsDisposed) return; + DisposeCore(true); + + GC.SuppressFinalize(this); + IsDisposed = true; + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecoderInfo.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecoderInfo.cs new file mode 100644 index 000000000..31b236234 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecoderInfo.cs @@ -0,0 +1,58 @@ +using Beutl.Media.Decoding; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +public sealed class AVFDecoderInfo(AVFDecodingExtension extension) : IDecoderInfo +{ + public string Name => "AVFoundation"; + + //https://learn.microsoft.com/ja-jp/windows/win32/medfound/supported-media-formats-in-media-foundation + public IEnumerable AudioExtensions() + { + yield return ".mp3"; + yield return ".wav"; + yield return ".m4a"; + yield return ".aac"; + yield return ".wma"; + yield return ".sami"; + yield return ".smi"; + yield return ".m4v"; + yield return ".mov"; + yield return ".mp4"; + yield return ".avi"; + yield return ".adts"; + yield return ".asf"; + yield return ".wmv"; + yield return ".3gp"; + yield return ".3gp2"; + yield return ".3gpp"; + } + + public MediaReader? Open(string file, MediaOptions options) + { + try + { + return new AVFReader(file, options, extension); + } + catch + { + return null; + } + } + + public IEnumerable VideoExtensions() + { + yield return ".mp4"; + yield return ".mov"; + yield return ".m4v"; + yield return ".avi"; + yield return ".wmv"; + yield return ".sami"; + yield return ".smi"; + yield return ".adts"; + yield return ".asf"; + yield return ".3gp"; + yield return ".3gp2"; + yield return ".3gpp"; + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecodingExtension.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecodingExtension.cs new file mode 100644 index 000000000..8d522eab3 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecodingExtension.cs @@ -0,0 +1,36 @@ +using Beutl.Extensibility; +using Beutl.Media.Decoding; +using MonoMac.AppKit; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +[Export] +public class AVFDecodingExtension : DecodingExtension +{ + public override string Name => "AVFoundation Decoding"; + + public override string DisplayName => "AVFoundation Decoding"; + + public override AVFDecodingSettings? Settings { get; } = new(); + + public override IDecoderInfo GetDecoderInfo() + { + return new AVFDecoderInfo(this); + } + + public override void Load() + { + if (OperatingSystem.IsMacOS()) + { + try + { + NSApplication.Init(); + } + catch + { + } + + DecoderRegistry.Register(GetDecoderInfo()); + } + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecodingSettings.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecodingSettings.cs new file mode 100644 index 000000000..9ac63a74a --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFDecodingSettings.cs @@ -0,0 +1,65 @@ +using System.ComponentModel.DataAnnotations; +using Beutl.Extensibility; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +public sealed class AVFDecodingSettings : ExtensionSettings +{ + public static readonly CoreProperty ThresholdFrameCountProperty; + public static readonly CoreProperty ThresholdSampleCountProperty; + public static readonly CoreProperty MaxVideoBufferSizeProperty; + public static readonly CoreProperty MaxAudioBufferSizeProperty; + + static AVFDecodingSettings() + { + ThresholdFrameCountProperty = ConfigureProperty(nameof(ThresholdFrameCount)) + .DefaultValue(30) + .Register(); + + ThresholdSampleCountProperty = ConfigureProperty(nameof(ThresholdSampleCount)) + .DefaultValue(30000) + .Register(); + + MaxVideoBufferSizeProperty = ConfigureProperty(nameof(MaxVideoBufferSize)) + .DefaultValue(4) + .Register(); + + MaxAudioBufferSizeProperty = ConfigureProperty(nameof(MaxAudioBufferSize)) + .DefaultValue(20) + .Register(); + + AffectsConfig( + ThresholdFrameCountProperty, + ThresholdSampleCountProperty, + MaxVideoBufferSizeProperty, + MaxAudioBufferSizeProperty); + } + + [Range(1, int.MaxValue)] + public int ThresholdFrameCount + { + get => GetValue(ThresholdFrameCountProperty); + set => SetValue(ThresholdFrameCountProperty, value); + } + + [Range(1, int.MaxValue)] + public int ThresholdSampleCount + { + get => GetValue(ThresholdSampleCountProperty); + set => SetValue(ThresholdSampleCountProperty, value); + } + + [Range(1, int.MaxValue)] + public int MaxVideoBufferSize + { + get => GetValue(MaxVideoBufferSizeProperty); + set => SetValue(MaxVideoBufferSizeProperty, value); + } + + [Range(1, int.MaxValue)] + public int MaxAudioBufferSize + { + get => GetValue(MaxAudioBufferSizeProperty); + set => SetValue(MaxAudioBufferSizeProperty, value); + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFReader.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFReader.cs new file mode 100644 index 000000000..9abf7069d --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFReader.cs @@ -0,0 +1,74 @@ +using System.Diagnostics.CodeAnalysis; +using Beutl.Media; +using Beutl.Media.Decoding; +using Beutl.Media.Music; +using MonoMac.AVFoundation; +using MonoMac.Foundation; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +public sealed class AVFReader : MediaReader +{ + private readonly AVAsset _asset; + + private AVFVideoStreamReader? _videoReader; + private AVFAudioStreamReader? _audioReader; + + public AVFReader(string file, MediaOptions options, AVFDecodingExtension extension) + { + var url = NSUrl.FromFilename(file); + _asset = AVAsset.FromUrl(url); + if (options.StreamsToLoad.HasFlag(MediaMode.Video)) + { + _videoReader = new AVFVideoStreamReader(_asset, extension); + } + + if (options.StreamsToLoad.HasFlag(MediaMode.Audio)) + { + _audioReader = new AVFAudioStreamReader(_asset, options, extension); + } + } + + public override VideoStreamInfo VideoInfo => + _videoReader?.VideoInfo ?? throw new Exception("VideoInfo is not available."); + + public override AudioStreamInfo AudioInfo => + _audioReader?.AudioInfo ?? throw new Exception("AudioInfo is not available."); + + public override bool HasVideo => _videoReader != null; + + public override bool HasAudio => _audioReader != null; + + public override bool ReadAudio(int start, int length, [NotNullWhen(true)] out IPcm? sound) + { + if (_audioReader != null) + { + return _audioReader.ReadAudio(start, length, out sound); + } + + sound = null; + return false; + } + + public override bool ReadVideo(int frame, [NotNullWhen(true)] out IBitmap? image) + { + if (_videoReader != null) + { + return _videoReader.ReadVideo(frame, out image); + } + + image = null; + return false; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + _audioReader?.Dispose(); + _videoReader?.Dispose(); + _asset.Dispose(); + + _audioReader = null; + _videoReader = null; + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFSampleCacheOptions.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFSampleCacheOptions.cs new file mode 100644 index 000000000..c8a689bd3 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFSampleCacheOptions.cs @@ -0,0 +1,5 @@ +namespace Beutl.Extensions.AVFoundation.Decoding; + +public record AVFSampleCacheOptions( + int MaxVideoBufferSize = 4, // あまり大きな値を設定するとReadSampleで停止する + int MaxAudioBufferSize = 20); diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFVideoSampleCache.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFVideoSampleCache.cs new file mode 100644 index 000000000..2b1cc2f10 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFVideoSampleCache.cs @@ -0,0 +1,74 @@ +using Beutl.Collections; +using Beutl.Logging; +using Microsoft.Extensions.Logging; +using MonoMac.CoreMedia; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +public class AVFVideoSampleCache(AVFSampleCacheOptions options) +{ + private readonly ILogger _logger = Log.CreateLogger(); + + public const int FrameWaringGapCount = 1; + + private CircularBuffer _buffer = new(options.MaxVideoBufferSize); + + private readonly record struct VideoCache(int Frame, CMSampleBuffer Sample); + + public void Reset() + { + CircularBuffer old = _buffer; + _buffer = new CircularBuffer(options.MaxVideoBufferSize); + foreach (VideoCache item in old) + { + item.Sample.Dispose(); + } + } + + public void Add(int frame, CMSampleBuffer pSample) + { + int lastFrameNum = LastFrameNumber(); + if (lastFrameNum != -1) + { + if (Math.Abs(lastFrameNum + 1 - frame) > FrameWaringGapCount) + { + _logger.LogWarning("frame error - frame: {frame} actual frame: {actual}", frame, lastFrameNum + 1); + } + + frame = lastFrameNum + 1; + } + + if (_buffer.IsFull) + { + _buffer.Front().Sample.Dispose(); + _buffer.PopFront(); + } + + var videoCache = new VideoCache(frame, pSample); + _buffer.PushBack(videoCache); + } + + public int LastFrameNumber() + { + if (_buffer.Size > 0) + { + VideoCache prevVideoCache = _buffer.Back(); + return prevVideoCache.Frame; + } + + return -1; + } + + public CMSampleBuffer? SearchSample(int frame) + { + foreach (VideoCache videoCache in _buffer.Reverse()) + { + if (videoCache.Frame == frame) + { + return videoCache.Sample; + } + } + + return null; + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Decoding/AVFVideoStreamReader.cs b/src/Beutl.Extensions.AVFoundation/Decoding/AVFVideoStreamReader.cs new file mode 100644 index 000000000..ddb4a38af --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Decoding/AVFVideoStreamReader.cs @@ -0,0 +1,191 @@ +using System.Diagnostics.CodeAnalysis; +using Beutl.Logging; +using Beutl.Media; +using Beutl.Media.Decoding; +using Microsoft.Extensions.Logging; +using MonoMac.AVFoundation; +using MonoMac.CoreMedia; +using MonoMac.CoreVideo; + +namespace Beutl.Extensions.AVFoundation.Decoding; + +public class AVFVideoStreamReader : IDisposable +{ + private readonly ILogger _logger = Log.CreateLogger(); + private readonly AVAsset _asset; + + private readonly AVFVideoSampleCache _sampleCache; + + // 現在のフレームからどれくらいの範囲ならシーケンシャル読み込みさせるかの閾値 + private readonly int _thresholdFrameCount; + + private readonly AVAssetTrack _track; + private AVAssetReader _reader; + private AVAssetReaderTrackOutput _output; + private CMTime _currentTimestamp; + + + public AVFVideoStreamReader(AVAsset asset, AVFDecodingExtension extension) + { + _asset = asset; + _sampleCache = new AVFVideoSampleCache( + new AVFSampleCacheOptions(MaxVideoBufferSize: extension.Settings?.MaxVideoBufferSize ?? 4)); + _thresholdFrameCount = extension.Settings?.ThresholdFrameCount ?? 30; + + _track = _asset.TracksWithMediaType(AVMediaType.Video)[0]; + + _reader = AVAssetReader.FromAsset(_asset, out var error); + if (error != null) throw new Exception(error.LocalizedDescription); + + _output = new AVAssetReaderTrackOutput( + _track, + new CVPixelBufferAttributes { PixelFormatType = CVPixelFormatType.CV32ARGB }.Dictionary); + _output.AlwaysCopiesSampleData = false; + + _reader.AddOutput(_output); + _reader.StartReading(); + + var desc = _track.FormatDescriptions[0]; + var frameSize = new PixelSize(desc.VideoDimensions.Width, desc.VideoDimensions.Height); + string codec = desc.VideoCodecType.ToString(); + float framerate = _track.NominalFrameRate; + double duration = _track.TotalSampleDataLength / _track.EstimatedDataRate * 8d; + VideoInfo = new VideoStreamInfo( + codec, + Rational.FromDouble(duration), + frameSize, + Rational.FromSingle(framerate)); + } + + ~AVFVideoStreamReader() + { + if (!IsDisposed) + { + DisposeCore(false); + } + } + + public VideoStreamInfo VideoInfo { get; } + + public bool IsDisposed { get; private set; } + + private CMSampleBuffer? ReadSample() + { + var buffer = _output.CopyNextSampleBuffer(); + if (!buffer.DataIsReady) + { + _logger.LogTrace("buffer.DataIsReady = false"); + return null; + } + + if (!buffer.IsValid) + { + _logger.LogTrace("buffer is invalid."); + return null; + } + + // success! + // add cache + // timestamp -= _firstGapTimeStamp; + _currentTimestamp = buffer.PresentationTimeStamp; + int frame = CMTimeUtilities.ConvertFrameFromTimeStamp(_currentTimestamp, _track.NominalFrameRate); + _sampleCache.Add(frame, buffer); + + return buffer; + } + + private void Seek(CMTime timestamp) + { + _sampleCache.Reset(); + _reader.Dispose(); + _output.Dispose(); + + _reader = AVAssetReader.FromAsset(_asset, out var error); + if (error != null) throw new Exception(error.LocalizedDescription); + _reader.TimeRange = new CMTimeRange { Start = timestamp, Duration = CMTime.PositiveInfinity }; + + _output = new AVAssetReaderTrackOutput( + _track, + new CVPixelBufferAttributes { PixelFormatType = CVPixelFormatType.CV32ARGB }.Dictionary); + _output.AlwaysCopiesSampleData = false; + _reader.AddOutput(_output); + + _reader.StartReading(); + } + + public bool ReadVideo(int frame, [NotNullWhen(true)] out IBitmap? image) + { + CMSampleBuffer? sample = _sampleCache.SearchSample(frame); + if (sample != null) + { + image = AVFSampleUtilities.ConvertToBgra(sample); + if (image != null) + return true; + } + + int currentFrame = _sampleCache.LastFrameNumber(); + + if (currentFrame == -1) + { + currentFrame = + CMTimeUtilities.ConvertFrameFromTimeStamp(_currentTimestamp, _track.NominalFrameRate); + } + + if (frame < currentFrame || (currentFrame + _thresholdFrameCount) < frame) + { + CMTime destTimePosition = CMTimeUtilities.ConvertTimeStampFromFrame(frame, _track.NominalFrameRate); + Seek(destTimePosition); + _logger.LogDebug( + "ReadFrame Seek currentFrame: {currentFrame}, destFrame: {destFrame} - destTimePos: {destTimePos} relativeFrame: {relativeFrame}", + currentFrame, frame, destTimePosition.Seconds, frame - currentFrame); + } + + sample = ReadSample(); + while (sample != null) + { + try + { + int readSampleFrame = _sampleCache.LastFrameNumber(); + + if (frame <= readSampleFrame) + { + if ((readSampleFrame - frame) > 0) + { + _logger.LogWarning( + "wrong frame currentFrame: {currentFrame} targetFrame: {frame} readSampleFrame: {readSampleFrame} distance: {distance}", + currentFrame, frame, readSampleFrame, readSampleFrame - frame); + } + + image = AVFSampleUtilities.ConvertToBgra(sample); + if (image != null) + return true; + } + + sample = ReadSample(); + } + catch + { + break; + } + } + + image = null; + return false; + } + + private void DisposeCore(bool disposing) + { + _sampleCache.Reset(); + _output.Dispose(); + _reader.Dispose(); + } + + public void Dispose() + { + if (IsDisposed) return; + DisposeCore(true); + + GC.SuppressFinalize(this); + IsDisposed = true; + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Encoding/AVFEncoderInfo.cs b/src/Beutl.Extensions.AVFoundation/Encoding/AVFEncoderInfo.cs new file mode 100644 index 000000000..da2353fd2 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Encoding/AVFEncoderInfo.cs @@ -0,0 +1,48 @@ +using System.Runtime.Versioning; +using Beutl.Media.Encoding; + +namespace Beutl.Extensions.AVFoundation.Encoding; + +[SupportedOSPlatform("macos")] +public sealed class AVFEncoderInfo(AVFEncodingExtension extension) : IEncoderInfo +{ + public string Name => "AVFoundation"; + + public MediaWriter? Create(string file, VideoEncoderSettings videoConfig, AudioEncoderSettings audioConfig) + { + try + { + return new AVFWriter(file, (AVFVideoEncoderSettings)videoConfig, (AVFAudioEncoderSettings)audioConfig); + } + catch (Exception e) + { + return null; + } + } + + public IEnumerable SupportExtensions() + { + yield return ".mp4"; + yield return ".mov"; + yield return ".m4v"; + yield return ".avi"; + yield return ".wmv"; + yield return ".sami"; + yield return ".smi"; + yield return ".adts"; + yield return ".asf"; + yield return ".3gp"; + yield return ".3gp2"; + yield return ".3gpp"; + } + + public VideoEncoderSettings DefaultVideoConfig() + { + return new AVFVideoEncoderSettings(); + } + + public AudioEncoderSettings DefaultAudioConfig() + { + return new AVFAudioEncoderSettings(); + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Encoding/AVFEncodingExtension.cs b/src/Beutl.Extensions.AVFoundation/Encoding/AVFEncodingExtension.cs new file mode 100644 index 000000000..fbe4b5a17 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Encoding/AVFEncodingExtension.cs @@ -0,0 +1,39 @@ +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.ObjectiveC; +using System.Runtime.Versioning; +using Beutl.Extensibility; +using Beutl.Media.Decoding; +using Beutl.Media.Encoding; +using MonoMac.AppKit; + +namespace Beutl.Extensions.AVFoundation.Encoding; + +[Export] +public class AVFEncodingExtension : EncodingExtension +{ + public override string Name => "AVFoundation Encoding"; + + public override string DisplayName => "AVFoundation Encoding"; + + [SupportedOSPlatform("macos")] + public override IEncoderInfo GetEncoderInfo() + { + return new AVFEncoderInfo(this); + } + + public override void Load() + { + if (OperatingSystem.IsMacOS()) + { + try + { + NSApplication.Init(); + } + catch + { + } + + EncoderRegistry.Register(GetEncoderInfo()); + } + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Encoding/AVFVideoEncoderSettings.cs b/src/Beutl.Extensions.AVFoundation/Encoding/AVFVideoEncoderSettings.cs new file mode 100644 index 000000000..96f237775 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Encoding/AVFVideoEncoderSettings.cs @@ -0,0 +1,211 @@ +using Beutl.Media.Encoding; + +namespace Beutl.Extensions.AVFoundation.Encoding; + +public sealed class AVFAudioEncoderSettings : AudioEncoderSettings +{ + public static readonly CoreProperty FormatProperty; + public static readonly CoreProperty LinearPcmBitDepthProperty; + public static readonly CoreProperty LinearPcmBigEndianProperty; + public static readonly CoreProperty LinearPcmFloatProperty; + public static readonly CoreProperty LinearPcmNonInterleavedProperty; + public static readonly CoreProperty QualityProperty; + public static readonly CoreProperty SampleRateConverterQualityProperty; + + static AVFAudioEncoderSettings() + { + FormatProperty = ConfigureProperty(nameof(Format)) + .DefaultValue(AudioFormatType.MPEG4AAC) + .Register(); + + LinearPcmBitDepthProperty = ConfigureProperty(nameof(LinearPcmBitDepth)) + .DefaultValue(BitDepth.Bits16) + .Register(); + + LinearPcmBigEndianProperty = ConfigureProperty(nameof(LinearPcmBigEndian)) + .DefaultValue(false) + .Register(); + + LinearPcmFloatProperty = ConfigureProperty(nameof(LinearPcmFloat)) + .DefaultValue(false) + .Register(); + + LinearPcmNonInterleavedProperty = + ConfigureProperty(nameof(LinearPcmNonInterleaved)) + .DefaultValue(false) + .Register(); + + QualityProperty = ConfigureProperty(nameof(Quality)) + .DefaultValue(AudioQuality.Default) + .Register(); + + SampleRateConverterQualityProperty = + ConfigureProperty(nameof(SampleRateConverterQuality)) + .DefaultValue(AudioQuality.Default) + .Register(); + + BitrateProperty.OverrideDefaultValue(-1); + } + + public AudioFormatType Format + { + get => GetValue(FormatProperty); + set => SetValue(FormatProperty, value); + } + + public BitDepth LinearPcmBitDepth + { + get => GetValue(LinearPcmBitDepthProperty); + set => SetValue(LinearPcmBitDepthProperty, value); + } + + public bool LinearPcmBigEndian + { + get => GetValue(LinearPcmBigEndianProperty); + set => SetValue(LinearPcmBigEndianProperty, value); + } + + public bool LinearPcmFloat + { + get => GetValue(LinearPcmFloatProperty); + set => SetValue(LinearPcmFloatProperty, value); + } + + public bool LinearPcmNonInterleaved + { + get => GetValue(LinearPcmNonInterleavedProperty); + set => SetValue(LinearPcmNonInterleavedProperty, value); + } + + public AudioQuality Quality + { + get => GetValue(QualityProperty); + set => SetValue(QualityProperty, value); + } + + public AudioQuality SampleRateConverterQuality + { + get => GetValue(SampleRateConverterQualityProperty); + set => SetValue(SampleRateConverterQualityProperty, value); + } + + public enum BitDepth + { + Bits8 = 8, + Bits16 = 16, + Bits24 = 24, + Bits32 = 32 + } + + public enum AudioQuality + { + Default = -1, + Min = 0, + Low = 32, // 0x00000020 + Medium = 64, // 0x00000040 + High = 96, // 0x00000060 + Max = 127, // 0x0000007F + } + + public enum AudioFormatType + { + MPEGLayer1 = 778924081, // 0x2E6D7031 + MPEGLayer2 = 778924082, // 0x2E6D7032 + MPEGLayer3 = 778924083, // 0x2E6D7033 + Audible = 1096107074, // 0x41554442 + MACE3 = 1296122675, // 0x4D414333 + MACE6 = 1296122678, // 0x4D414336 + QDesign2 = 1363430706, // 0x51444D32 + QDesign = 1363430723, // 0x51444D43 + QUALCOMM = 1365470320, // 0x51636C70 + MPEG4AAC = 1633772320, // 0x61616320 + MPEG4AAC_ELD = 1633772389, // 0x61616365 + MPEG4AAC_ELD_SBR = 1633772390, // 0x61616366 + MPEG4AAC_ELD_V2 = 1633772391, // 0x61616367 + MPEG4AAC_HE = 1633772392, // 0x61616368 + MPEG4AAC_LD = 1633772396, // 0x6161636C + MPEG4AAC_HE_V2 = 1633772400, // 0x61616370 + MPEG4AAC_Spatial = 1633772403, // 0x61616373 + AC3 = 1633889587, // 0x61632D33 + AES3 = 1634038579, // 0x61657333 + AppleLossless = 1634492771, // 0x616C6163 + ALaw = 1634492791, // 0x616C6177 + ParameterValueStream = 1634760307, // 0x61707673 + CAC3 = 1667326771, // 0x63616333 + MPEG4CELP = 1667591280, // 0x63656C70 + MPEG4HVXC = 1752594531, // 0x68767863 + iLBC = 1768710755, // 0x696C6263 + AppleIMA4 = 1768775988, // 0x696D6134 + LinearPCM = 1819304813, // 0x6C70636D + MIDIStream = 1835623529, // 0x6D696469 + DVIIntelIMA = 1836253201, // 0x6D730011 + MicrosoftGSM = 1836253233, // 0x6D730031 + AMR = 1935764850, // 0x73616D72 + TimeCode = 1953066341, // 0x74696D65 + MPEG4TwinVQ = 1953986161, // 0x74777671 + ULaw = 1970037111, // 0x756C6177 + } +} + +public sealed class AVFVideoEncoderSettings : VideoEncoderSettings +{ + public static readonly CoreProperty CodecProperty; + public static readonly CoreProperty JPEGQualityProperty; + public static readonly CoreProperty ProfileLevelH264Property; + + static AVFVideoEncoderSettings() + { + CodecProperty = ConfigureProperty(nameof(Codec)) + .DefaultValue(VideoCodec.H264) + .Register(); + + JPEGQualityProperty = ConfigureProperty(nameof(JPEGQuality)) + .DefaultValue(-1) + .Register(); + + ProfileLevelH264Property = + ConfigureProperty(nameof(ProfileLevelH264)) + .DefaultValue(VideoProfileLevelH264.Default) + .Register(); + + BitrateProperty.OverrideDefaultValue(-1); + KeyframeRateProperty.OverrideDefaultValue(-1); + } + + public VideoCodec Codec + { + get => GetValue(CodecProperty); + set => SetValue(CodecProperty, value); + } + + public float JPEGQuality + { + get => GetValue(JPEGQualityProperty); + set => SetValue(JPEGQualityProperty, value); + } + + public VideoProfileLevelH264 ProfileLevelH264 + { + get => GetValue(ProfileLevelH264Property); + set => SetValue(ProfileLevelH264Property, value); + } + + public enum VideoCodec + { + Default = 0, + H264 = 1, + JPEG = 2, + } + + public enum VideoProfileLevelH264 + { + Default = 0, + Baseline30 = 1, + Baseline31 = 2, + Baseline41 = 3, + Main30 = 4, + Main31 = 5, + Main32 = 6, + Main41 = 7, + } +} diff --git a/src/Beutl.Extensions.AVFoundation/Encoding/AVFWriter.cs b/src/Beutl.Extensions.AVFoundation/Encoding/AVFWriter.cs new file mode 100644 index 000000000..81e6013d2 --- /dev/null +++ b/src/Beutl.Extensions.AVFoundation/Encoding/AVFWriter.cs @@ -0,0 +1,342 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using Beutl.Media; +using Beutl.Media.Encoding; +using Beutl.Media.Music; +using Beutl.Media.Music.Samples; +using Beutl.Media.Pixel; +using MonoMac.AudioToolbox; +using MonoMac.AVFoundation; +using MonoMac.CoreMedia; +using MonoMac.CoreVideo; +using MonoMac.Foundation; + +namespace Beutl.Extensions.AVFoundation.Encoding; + +[SupportedOSPlatform("macos")] +public class AVFWriter : MediaWriter +{ + private readonly AVAssetWriter _assetWriter; + private readonly AVAssetWriterInput _videoInput; + private readonly AVAssetWriterInputPixelBufferAdaptor _videoAdaptor; + private long _numberOfFrames; + private readonly AVAssetWriterInput _audioInput; + private long _numberOfSamples; + + public AVFWriter(string file, AVFVideoEncoderSettings videoConfig, AVFAudioEncoderSettings audioConfig) + : base(videoConfig, audioConfig) + { + var url = NSUrl.FromFilename(file); + _assetWriter = AVAssetWriter.FromUrl(url, AVFileType.Mpeg4, out var error); + if (error != null) throw new Exception(error.LocalizedDescription); + + _videoInput = AVAssetWriterInput.Create(AVMediaType.Video, new AVVideoSettingsCompressed + { + Width = videoConfig.DestinationSize.Width, + Height = videoConfig.DestinationSize.Height, + Codec = ToAVVideoCodec(videoConfig.Codec), + CodecSettings = new AVVideoCodecSettings + { + AverageBitRate = videoConfig.Bitrate == -1 ? null : videoConfig.Bitrate, + MaxKeyFrameInterval = videoConfig.KeyframeRate == -1 ? null : videoConfig.KeyframeRate, + JPEGQuality = videoConfig.JPEGQuality < 0 ? null : videoConfig.JPEGQuality, + ProfileLevelH264 = ToAVVideoProfileLevelH264(videoConfig.ProfileLevelH264), + }, + }); + _videoInput.ExpectsMediaDataInRealTime = true; + _videoAdaptor = AVAssetWriterInputPixelBufferAdaptor.Create(_videoInput, + new CVPixelBufferAttributes + { + PixelFormatType = CVPixelFormatType.CV32ARGB, + Width = videoConfig.SourceSize.Width, + Height = videoConfig.SourceSize.Width, + }); + _assetWriter.AddInput(_videoInput); + + var audioSettings = new AudioSettings + { + SampleRate = audioConfig.SampleRate, + EncoderBitRate = audioConfig.Bitrate == -1 ? null : audioConfig.Bitrate, + NumberChannels = audioConfig.Channels, + Format = ToAudioFormatType(audioConfig.Format), + AudioQuality = + audioConfig.Quality == AVFAudioEncoderSettings.AudioQuality.Default + ? null + : (AVAudioQuality?)audioConfig.Quality, + SampleRateConverterAudioQuality = + audioConfig.SampleRateConverterQuality == AVFAudioEncoderSettings.AudioQuality.Default + ? null + : (AVAudioQuality?)audioConfig.SampleRateConverterQuality, + }; + if (audioSettings.Format == AudioFormatType.LinearPCM) + { + audioSettings.LinearPcmFloat = audioConfig.LinearPcmFloat; + audioSettings.LinearPcmBigEndian = audioConfig.LinearPcmBigEndian; + audioSettings.LinearPcmBitDepth = (int?)audioConfig.LinearPcmBitDepth; + audioSettings.LinearPcmNonInterleaved = audioConfig.LinearPcmNonInterleaved; + } + + _audioInput = AVAssetWriterInput.Create(AVMediaType.Audio, audioSettings); + _audioInput.ExpectsMediaDataInRealTime = true; + _assetWriter.AddInput(_audioInput); + + if (!_assetWriter.StartWriting()) + { + throw new Exception("Failed to start writing"); + } + + _assetWriter.StartSessionAtSourceTime(CMTime.Zero); + } + + public override long NumberOfFrames => _numberOfFrames; + + public override long NumberOfSamples => _numberOfSamples; + + public override bool AddVideo(IBitmap image) + { + int count = 0; + while (!_videoAdaptor.AssetWriterInput.ReadyForMoreMediaData) + { + Thread.Sleep(10); + count++; + if (count > 100) + { + return false; + } + } + + var time = new CMTime(_numberOfFrames * VideoConfig.FrameRate.Denominator, + (int)VideoConfig.FrameRate.Numerator); + CVPixelBuffer? pixelBuffer; + if (image is Bitmap bgra8888) + { + pixelBuffer = AVFSampleUtilities.ConvertToCVPixelBuffer(bgra8888); + } + else + { + using var copy = image.Convert(); + pixelBuffer = AVFSampleUtilities.ConvertToCVPixelBuffer(copy); + } + + if (pixelBuffer == null) + { + return false; + } + + if (!_videoAdaptor.AppendPixelBufferWithPresentationTime(pixelBuffer, time)) + { + return false; + } + + _numberOfFrames++; + return true; + } + + [DllImport("/System/Library/PrivateFrameworks/CoreMedia.framework/Versions/A/CoreMedia")] + private static extern unsafe CMFormatDescriptionError CMAudioFormatDescriptionCreate( + IntPtr allocator, + void* asbd, + uint layoutSize, + void* layout, + uint magicCookieSize, + void* magicCookie, + IntPtr extensions, + out IntPtr handle); + + [UnsafeAccessor(UnsafeAccessorKind.Constructor)] + private static extern CMAudioFormatDescription NewCMAudioFormatDescription(IntPtr handle); + + [UnsafeAccessor(UnsafeAccessorKind.Constructor)] + private static extern CMBlockBuffer NewCMBlockBuffer(IntPtr handle); + + [DllImport("/System/Library/PrivateFrameworks/CoreMedia.framework/Versions/A/CoreMedia")] + private static extern CMBlockBufferError CMBlockBufferCreateWithMemoryBlock( + IntPtr allocator, + IntPtr memoryBlock, + uint blockLength, + IntPtr blockAllocator, + IntPtr customBlockSource, + uint offsetToData, + uint dataLength, + CMBlockBufferFlags flags, + out IntPtr handle); + + private static unsafe CMAudioFormatDescription CreateAudioFormatDescription(AudioStreamBasicDescription asbd) + { + var channelLayout = new AudioChannelLayout + { + AudioTag = asbd.ChannelsPerFrame == 2 ? AudioChannelLayoutTag.Stereo : AudioChannelLayoutTag.Mono, + Channels = [], + Bitmap = 0, + }; + var data = channelLayout.AsData(); + + var error = CMAudioFormatDescriptionCreate( + IntPtr.Zero, + &asbd, + (uint)data.Length, + (void*)data.Bytes, + 0, + null, + IntPtr.Zero, + out var handle); + if (error != CMFormatDescriptionError.None) throw new Exception(error.ToString()); + return NewCMAudioFormatDescription(handle); + } + + private static CMBlockBuffer CreateCMBlockBufferWithMemoryBlock(uint length, IntPtr memoryBlock, + CMBlockBufferFlags flags) + { + var error = CMBlockBufferCreateWithMemoryBlock( + IntPtr.Zero, + memoryBlock, + length, + IntPtr.Zero, + IntPtr.Zero, + 0, + length, + flags, + out var handle); + if (error != CMBlockBufferError.None) throw new Exception(error.ToString()); + return NewCMBlockBuffer(handle); + } + + public override bool AddAudio(IPcm sound) + { + int count = 0; + while (!_audioInput.ReadyForMoreMediaData) + { + Thread.Sleep(10); + count++; + if (count > 100) + { + return false; + } + } + + var sourceFormat = AudioStreamBasicDescription.CreateLinearPCM(sound.SampleRate, (uint)sound.NumChannels); + sourceFormat.FormatFlags = GetFormatFlags(); + sourceFormat.BitsPerChannel = GetBits(); + var fmtError = AudioStreamBasicDescription.GetFormatInfo(ref sourceFormat); + if (fmtError != AudioFormatError.None) throw new Exception(fmtError.ToString()); + + uint inputDataSize = (uint)(sound.SampleSize * sound.NumSamples); + var time = new CMTime(_numberOfSamples, sound.SampleRate); + var dataBuffer = CreateCMBlockBufferWithMemoryBlock( + inputDataSize, sound.Data, CMBlockBufferFlags.AlwaysCopyData); + + var formatDescription = CreateAudioFormatDescription(sourceFormat); + + var sampleBuffer = CMSampleBuffer.CreateWithPacketDescriptions(dataBuffer, formatDescription, + sound.NumSamples, time, null, out var error4); + if (error4 != CMSampleBufferError.None) throw new Exception(error4.ToString()); + + if (!_audioInput.AppendSampleBuffer(sampleBuffer)) + { + return false; + } + + _numberOfSamples += sound.NumSamples; + return true; + + int GetBits() + { + return sound switch + { + Pcm or Pcm => 32, + Pcm => 16, + _ => throw new NotSupportedException() + }; + } + + AudioFormatFlags GetFormatFlags() + { + return sound switch + { + Pcm => AudioFormatFlags.IsFloat | AudioFormatFlags.IsPacked, + Pcm or Pcm => AudioFormatFlags.IsSignedInteger | + AudioFormatFlags.IsPacked, + _ => throw new NotSupportedException() + }; + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + _videoInput.MarkAsFinished(); + _audioInput.MarkAsFinished(); + _assetWriter.EndSessionAtSourceTime(new CMTime(_numberOfFrames * VideoConfig.FrameRate.Denominator, + (int)VideoConfig.FrameRate.Numerator)); + _assetWriter.FinishWriting(); + } + + private AVVideoCodec? ToAVVideoCodec(AVFVideoEncoderSettings.VideoCodec codec) + { + return codec switch + { + AVFVideoEncoderSettings.VideoCodec.H264 => AVVideoCodec.H264, + AVFVideoEncoderSettings.VideoCodec.JPEG => AVVideoCodec.JPEG, + _ => null + }; + } + + private AVVideoProfileLevelH264? ToAVVideoProfileLevelH264(AVFVideoEncoderSettings.VideoProfileLevelH264 profile) + { + return profile switch + { + AVFVideoEncoderSettings.VideoProfileLevelH264.Baseline30 => AVVideoProfileLevelH264.Baseline30, + AVFVideoEncoderSettings.VideoProfileLevelH264.Baseline31 => AVVideoProfileLevelH264.Baseline31, + AVFVideoEncoderSettings.VideoProfileLevelH264.Baseline41 => AVVideoProfileLevelH264.Baseline41, + AVFVideoEncoderSettings.VideoProfileLevelH264.Main30 => AVVideoProfileLevelH264.Main30, + AVFVideoEncoderSettings.VideoProfileLevelH264.Main31 => AVVideoProfileLevelH264.Main31, + AVFVideoEncoderSettings.VideoProfileLevelH264.Main32 => AVVideoProfileLevelH264.Main32, + AVFVideoEncoderSettings.VideoProfileLevelH264.Main41 => AVVideoProfileLevelH264.Main41, + _ => null + }; + } + + private AudioFormatType? ToAudioFormatType(AVFAudioEncoderSettings.AudioFormatType format) + { + return format switch + { + AVFAudioEncoderSettings.AudioFormatType.MPEGLayer1 => AudioFormatType.MPEGLayer1, + AVFAudioEncoderSettings.AudioFormatType.MPEGLayer2 => AudioFormatType.MPEGLayer2, + AVFAudioEncoderSettings.AudioFormatType.MPEGLayer3 => AudioFormatType.MPEGLayer3, + AVFAudioEncoderSettings.AudioFormatType.Audible => AudioFormatType.Audible, + AVFAudioEncoderSettings.AudioFormatType.MACE3 => AudioFormatType.MACE3, + AVFAudioEncoderSettings.AudioFormatType.MACE6 => AudioFormatType.MACE6, + AVFAudioEncoderSettings.AudioFormatType.QDesign2 => AudioFormatType.QDesign2, + AVFAudioEncoderSettings.AudioFormatType.QDesign => AudioFormatType.QDesign, + AVFAudioEncoderSettings.AudioFormatType.QUALCOMM => AudioFormatType.QUALCOMM, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC => AudioFormatType.MPEG4AAC, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC_ELD => AudioFormatType.MPEG4AAC_ELD, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC_ELD_SBR => AudioFormatType.MPEG4AAC_ELD_SBR, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC_ELD_V2 => AudioFormatType.MPEG4AAC_ELD_V2, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC_HE => AudioFormatType.MPEG4AAC_HE, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC_LD => AudioFormatType.MPEG4AAC_LD, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC_HE_V2 => AudioFormatType.MPEG4AAC_HE_V2, + AVFAudioEncoderSettings.AudioFormatType.MPEG4AAC_Spatial => AudioFormatType.MPEG4AAC_Spatial, + AVFAudioEncoderSettings.AudioFormatType.AC3 => AudioFormatType.AC3, + AVFAudioEncoderSettings.AudioFormatType.AES3 => AudioFormatType.AES3, + AVFAudioEncoderSettings.AudioFormatType.AppleLossless => AudioFormatType.AppleLossless, + AVFAudioEncoderSettings.AudioFormatType.ALaw => AudioFormatType.ALaw, + AVFAudioEncoderSettings.AudioFormatType.ParameterValueStream => AudioFormatType.ParameterValueStream, + AVFAudioEncoderSettings.AudioFormatType.CAC3 => AudioFormatType.CAC3, + AVFAudioEncoderSettings.AudioFormatType.MPEG4CELP => AudioFormatType.MPEG4CELP, + AVFAudioEncoderSettings.AudioFormatType.MPEG4HVXC => AudioFormatType.MPEG4HVXC, + AVFAudioEncoderSettings.AudioFormatType.iLBC => AudioFormatType.iLBC, + AVFAudioEncoderSettings.AudioFormatType.AppleIMA4 => AudioFormatType.AppleIMA4, + AVFAudioEncoderSettings.AudioFormatType.LinearPCM => AudioFormatType.LinearPCM, + AVFAudioEncoderSettings.AudioFormatType.MIDIStream => AudioFormatType.MIDIStream, + AVFAudioEncoderSettings.AudioFormatType.DVIIntelIMA => AudioFormatType.DVIIntelIMA, + AVFAudioEncoderSettings.AudioFormatType.MicrosoftGSM => AudioFormatType.MicrosoftGSM, + AVFAudioEncoderSettings.AudioFormatType.AMR => AudioFormatType.AMR, + AVFAudioEncoderSettings.AudioFormatType.TimeCode => AudioFormatType.TimeCode, + AVFAudioEncoderSettings.AudioFormatType.MPEG4TwinVQ => AudioFormatType.MPEG4TwinVQ, + AVFAudioEncoderSettings.AudioFormatType.ULaw => AudioFormatType.ULaw, + _ => null + }; + } +} diff --git a/src/Beutl/Beutl.csproj b/src/Beutl/Beutl.csproj index 376295be7..0ea888920 100644 --- a/src/Beutl/Beutl.csproj +++ b/src/Beutl/Beutl.csproj @@ -99,6 +99,11 @@ Include="..\Beutl.Extensions.MediaFoundation\Beutl.Extensions.MediaFoundation.csproj" /> + + + + TextTemplatingFileGenerator diff --git a/src/Beutl/Services/StartupTasks/LoadPrimitiveExtensionTask.cs b/src/Beutl/Services/StartupTasks/LoadPrimitiveExtensionTask.cs index 72ba33c6c..9f80d2284 100644 --- a/src/Beutl/Services/StartupTasks/LoadPrimitiveExtensionTask.cs +++ b/src/Beutl/Services/StartupTasks/LoadPrimitiveExtensionTask.cs @@ -6,6 +6,7 @@ namespace Beutl.Services.StartupTasks; public sealed class LoadPrimitiveExtensionTask : StartupTask { private readonly PackageManager _manager; + public static readonly Extension[] PrimitiveExtensions = [ EditPageExtension.Instance, @@ -40,6 +41,7 @@ public LoadPrimitiveExtensionTask(PackageManager manager) _manager.SetupExtensionSettings(item); item.Load(); } + provider.AddExtensions(LocalPackage.Reserved0, PrimitiveExtensions); activity?.AddEvent(new("Loaded_Extensions")); @@ -56,7 +58,16 @@ public LoadPrimitiveExtensionTask(PackageManager manager) Name = "Beutl.Embedding.FFmpeg", DisplayName = "Beutl.Embedding.FFmpeg", InstalledPath = AppContext.BaseDirectory, - Tags = { "ffmpeg", "decoder", "decoding", "encoder", "encoding", "video", "audio" }, + Tags = + { + "ffmpeg", + "decoder", + "decoding", + "encoder", + "encoding", + "video", + "audio" + }, Version = GitVersionInformation.NuGetVersionV2, WebSite = "https://github.com/b-editor/beutl", Publisher = "b-editor" @@ -95,7 +106,8 @@ public LoadPrimitiveExtensionTask(PackageManager manager) Name = "Beutl.Embedding.MediaFoundation", DisplayName = "Beutl.Embedding.MediaFoundation", InstalledPath = AppContext.BaseDirectory, - Tags = { "windows", "media-foundation", "decoder", "decoding", "encoder", "encoding", "video", "audio" }, + Tags = + { "windows", "media-foundation", "decoder", "decoding", "encoder", "encoding", "video", "audio" }, Version = GitVersionInformation.NuGetVersionV2, WebSite = "https://github.com/b-editor/beutl", Publisher = "b-editor" @@ -117,6 +129,53 @@ public LoadPrimitiveExtensionTask(PackageManager manager) } #pragma warning restore CS0436 #endif + +#pragma warning disable CS0436 + if (OperatingSystem.IsMacOS()) + { + activity?.AddEvent(new("Loading_AVFoundation")); + + // Beutl.Extensions.FFmpeg.csproj + var pkg = new LocalPackage + { + ShortDescription = "AVFoundation for beutl", + Name = "Beutl.Embedding.AVFoundation", + DisplayName = "Beutl.Embedding.AVFoundation", + InstalledPath = AppContext.BaseDirectory, + Tags = + { + "macos", + "avfoundation", + "decoder", + "decoding", + "encoder", + "encoding", + "video", + "audio" + }, + Version = GitVersionInformation.NuGetVersionV2, + WebSite = "https://github.com/b-editor/beutl", + Publisher = "b-editor" + }; + try + { + var decoding = new Extensions.AVFoundation.Decoding.AVFDecodingExtension(); + var encoding = new Extensions.AVFoundation.Encoding.AVFEncodingExtension(); + _manager.SetupExtensionSettings(decoding); + _manager.SetupExtensionSettings(encoding); + decoding.Load(); + encoding.Load(); + + provider.AddExtensions(pkg.LocalId, [decoding, encoding]); + } + catch (Exception ex) + { + Failures.Add((pkg, ex)); + } + + activity?.AddEvent(new("Loaded_AVFoundation")); + } +#pragma warning restore CS0436 } }); }