diff --git a/Limelight/Stream/Connection.m b/Limelight/Stream/Connection.m index 7cc84e3d..95e0506e 100644 --- a/Limelight/Stream/Connection.m +++ b/Limelight/Stream/Connection.m @@ -55,14 +55,9 @@ int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void* return 0; } -void DrStart(void) +void DrCleanup(void) { - [renderer start]; -} - -void DrStop(void) -{ - [renderer stop]; + [renderer cleanup]; } -(BOOL) getVideoStats:(video_stats_t*)stats @@ -433,9 +428,9 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer LiInitializeVideoCallbacks(&_drCallbacks); _drCallbacks.setup = DrDecoderSetup; - _drCallbacks.start = DrStart; - _drCallbacks.stop = DrStop; - _drCallbacks.capabilities = CAPABILITY_PULL_RENDERER | CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; + _drCallbacks.cleanup = DrCleanup; + _drCallbacks.submitDecodeUnit = DrSubmitDecodeUnit; + _drCallbacks.capabilities = CAPABILITY_DIRECT_SUBMIT | CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; LiInitializeAudioCallbacks(&_arCallbacks); _arCallbacks.init = ArInit; diff --git a/Limelight/Stream/VideoDecoderRenderer.h b/Limelight/Stream/VideoDecoderRenderer.h index 768db534..aeeabc28 100644 --- a/Limelight/Stream/VideoDecoderRenderer.h +++ b/Limelight/Stream/VideoDecoderRenderer.h @@ -15,9 +15,10 @@ - (id)initWithView:(UIView*)view callbacks:(id)callbacks streamAspectRatio:(float)aspectRatio useFramePacing:(BOOL)useFramePacing; - (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate; -- (void)start; -- (void)stop; +- (void)cleanup; - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(int)bufferType frameType:(int)frameType pts:(unsigned int)pts; +- (OSStatus)decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameType:(int)frameType; + @end diff --git a/Limelight/Stream/VideoDecoderRenderer.m b/Limelight/Stream/VideoDecoderRenderer.m index 7333c650..a4c6f396 100644 --- a/Limelight/Stream/VideoDecoderRenderer.m +++ b/Limelight/Stream/VideoDecoderRenderer.m @@ -6,6 +6,8 @@ // Copyright (c) 2014 Moonlight Stream. All rights reserved. // +@import VideoToolbox; + #import "VideoDecoderRenderer.h" #import "StreamView.h" @@ -23,6 +25,8 @@ @implementation VideoDecoderRenderer { NSData *spsData, *ppsData, *vpsData; CMVideoFormatDescriptionRef formatDesc; + CMVideoFormatDescriptionRef formatDescImageBuffer; + VTDecompressionSessionRef decompressionSession; CADisplayLink* _displayLink; BOOL framePacing; @@ -74,6 +78,17 @@ - (void)reinitializeDisplayLayer CFRelease(formatDesc); formatDesc = nil; } + + if (formatDescImageBuffer != nil) { + CFRelease(formatDescImageBuffer); + formatDescImageBuffer = nil; + } + + if (decompressionSession != nil){ + VTDecompressionSessionInvalidate(decompressionSession); + CFRelease(decompressionSession); + decompressionSession = nil; + } } - (id)initWithView:(StreamView*)view callbacks:(id)callbacks streamAspectRatio:(float)aspectRatio useFramePacing:(BOOL)useFramePacing @@ -94,10 +109,7 @@ - (void)setupWithVideoFormat:(int)videoFormat frameRate:(int)frameRate { self->videoFormat = videoFormat; self->frameRate = frameRate; -} - -- (void)start -{ + _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)]; if (@available(iOS 15.0, tvOS 15.0, *)) { _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->frameRate, self->frameRate, self->frameRate); @@ -106,6 +118,26 @@ - (void)start _displayLink.preferredFramesPerSecond = self->frameRate; } [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode]; + +} + +- (void) setupDecompressionSession { + if (decompressionSession != NULL){ + VTDecompressionSessionInvalidate(decompressionSession); + CFRelease(decompressionSession); + decompressionSession = nil; + } + + int status = VTDecompressionSessionCreate(kCFAllocatorDefault, + formatDesc, + nil, + nil, + nil, + &decompressionSession); + if (status != noErr) { + Log(LOG_E, @"Failed to instance VTDecompressionSessionRef, status %d", status); + } + } // TODO: Refactor this @@ -113,31 +145,10 @@ - (void)start - (void)displayLinkCallback:(CADisplayLink *)sender { - VIDEO_FRAME_HANDLE handle; - PDECODE_UNIT du; - - while (LiPollNextVideoFrame(&handle, &du)) { - LiCompleteVideoFrame(handle, DrSubmitDecodeUnit(du)); - - if (framePacing) { - // Calculate the actual display refresh rate - double displayRefreshRate = 1 / (_displayLink.targetTimestamp - _displayLink.timestamp); - - // Only pace frames if the display refresh rate is >= 90% of our stream frame rate. - // Battery saver, accessibility settings, or device thermals can cause the actual - // refresh rate of the display to drop below the physical maximum. - if (displayRefreshRate >= frameRate * 0.9f) { - // Keep one pending frame to smooth out gaps due to - // network jitter at the cost of 1 frame of latency - if (LiGetPendingVideoFrames() == 1) { - break; - } - } - } - } + // Do Nothing } -- (void)stop +- (void)cleanup { [_displayLink invalidate]; } @@ -262,6 +273,8 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i formatDesc = NULL; } } + + [self setupDecompressionSession]; } // Data is NOT to be freed here. It's a direct usage of the caller's buffer. @@ -330,11 +343,12 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i CMSampleBufferRef sampleBuffer; - status = CMSampleBufferCreate(kCFAllocatorDefault, + CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, CMTimeMake(pts, 1000), kCMTimeInvalid}; + + status = CMSampleBufferCreateReady(kCFAllocatorDefault, frameBlockBuffer, - true, NULL, - NULL, formatDesc, 1, 0, - NULL, 0, NULL, + formatDesc, 1, 1, + &sampleTiming, 0, NULL, &sampleBuffer); if (status != noErr) { Log(LOG_E, @"CMSampleBufferCreate failed: %d", (int)status); @@ -342,32 +356,12 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i CFRelease(frameBlockBuffer); return DR_NEED_IDR; } + + OSStatus decodeStatus = [self decodeFrameWithSampleBuffer: sampleBuffer frameType: frameType]; - CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); - CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); - - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_IsDependedOnByOthers, kCFBooleanTrue); - - if (frameType == FRAME_TYPE_PFRAME) { - // P-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanTrue); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanTrue); - } else { - // I-frame - CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse); - } - - // Enqueue the next frame - [self->displayLayer enqueueSampleBuffer:sampleBuffer]; - - if (frameType == FRAME_TYPE_IDR) { - // Ensure the layer is visible now - self->displayLayer.hidden = NO; - - // Tell our parent VC to hide the progress indicator - [self->_callbacks videoContentShown]; + if (decodeStatus != noErr){ + Log(LOG_E, @"Failed to decompress frame: %d", decodeStatus); + return DR_NEED_IDR; } // Dereference the buffers @@ -378,4 +372,52 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i return DR_OK; } +- (OSStatus) decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameType:(int)frameType{ + VTDecodeFrameFlags flags = kVTDecodeFrame_EnableAsynchronousDecompression; + + return VTDecompressionSessionDecodeFrameWithOutputHandler(decompressionSession, sampleBuffer, flags, NULL, ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) { + if (status != noErr) + { + NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil]; + Log(LOG_E, @"Decompression session error: %@", error); + LiRequestIdrFrame(); + return; + } + + if (self->formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->formatDescImageBuffer, imageBuffer)){ + + OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->formatDescImageBuffer)); + if (res != noErr){ + Log(LOG_E, @"Failed to create video format description from imageBuffer"); + return; + } + } + + CMSampleBufferRef sampleBuffer; + CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration}; + + OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->formatDescImageBuffer, &sampleTiming, &sampleBuffer); + + if (err != noErr){ + Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err); + return; + } + + // Enqueue the next frame + [self->displayLayer enqueueSampleBuffer:sampleBuffer]; + + dispatch_async(dispatch_get_main_queue(), ^{ + if (frameType == FRAME_TYPE_IDR) { + // Ensure the layer is visible now + self->displayLayer.hidden = NO; + + // Tell our parent VC to hide the progress indicator + [self->_callbacks videoContentShown]; + } + }); + + CFRelease(sampleBuffer); + }); +} + @end