Processed CMSampleBuffers come out black on weak(?) machines

Hi,

In our app we process frames captured from locally attached cameras to overlay static images and text. It is all standard AVFoundation calls like AVCaptureSession etc.

From the captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection callback we call the following method to process the captured sample buffer before calling [self.videoWriterInput appendSampleBuffer: sampleBuffer] to write it to disk:

- (CMSampleBufferRef) getProcessedVideoSample: (CMSampleBufferRef) sampleBuffer
{
	CMSampleBufferRef newSamplelBuf = nil;

	@autoreleasepool
	{
		CMItemCount numSamples = CMSampleBufferGetNumSamples( sampleBuffer );

		if (numSamples > 0)
		{
			CVImageBufferRef cvImageBuffer = CMSampleBufferGetImageBuffer( sampleBuffer );

			CMSampleTimingInfo sampleTiming = {
				.duration = CMSampleBufferGetDuration( sampleBuffer ),
				.presentationTimeStamp = CMSampleBufferGetPresentationTimeStamp( sampleBuffer ),
				.decodeTimeStamp = CMSampleBufferGetDecodeTimeStamp( sampleBuffer )
			};

			CIImage *videoFrameImage = [CIImage imageWithCVPixelBuffer: cvImageBuffer];

			CIImage *frameCIImage = nil;

			CVReturn error = kCVReturnError;

			Float64 escTimeSecs = CMTimeGetSeconds( sampleTiming.presentationTimeStamp );

			long frameMatchMillis = [self.matchDescription matchTimeInMillisForActualTime: escTimeSecs];
			
			CGImageRef baseOverlayImg = nil;

			@synchronized (self.liveOverlaysProject.staticImg_Lock) {

				baseOverlayImg = [self.liveOverlaysProject staticCGImageFor: frameMatchMillis];
				if (baseOverlayImg != nil)
				{
					CGImageRetain( baseOverlayImg );
				}
			}

			if (baseOverlayImg != nil)
			{
				CIImage *overCIImage = [[CIImage alloc] initWithCGImage: baseOverlayImg];

				[self.mtkCompositingFilter0 setValue: overCIImage forKey: @"inputImage"];
				[self.mtkCompositingFilter0 setValue: videoFrameImage forKey: @"inputBackgroundImage"];

				frameCIImage = self.mtkCompositingFilter0.outputImage;

				CGImageRelease( baseOverlayImg );
				baseOverlayImg = nil;
			}

			CIImage *CIImage_1 = nil;
			CIImage *CIImage_2 = nil;

			if (frameCIImage == nil)
				CIImage_1 = videoFrameImage;
			else
				CIImage_1 = frameCIImage;

			CGImageRef timeDependentImg = nil;

			@synchronized (self.liveOverlaysProject.liveImg_Lock) {

				timeDependentImg = [self.liveOverlaysProject liveCGImageFor: frameMatchMillis];
				if (timeDependentImg != nil)
				{
					CGImageRetain( timeDependentImg );
				}
			}

			if (timeDependentImg != nil)
			{
				CIImage *overCIImage = [[CIImage alloc] initWithCGImage: timeDependentImg];

				[self.mtkCompositingFilter1 setValue: overCIImage forKey: @"inputImage"];
				[self.mtkCompositingFilter1 setValue: CIImage_1 forKey: @"inputBackgroundImage"];

				CIImage_2 = self.mtkCompositingFilter1.outputImage;

				CGImageRelease( timeDependentImg );
				timeDependentImg = nil;
			}

			CIImage *finalCIImage = nil;

			if (CIImage_2 == nil)
				finalCIImage = CIImage_1;
			else
				finalCIImage = CIImage_2;

			CVPixelBufferRef outPixelBuf = NULL;

			if (self.cvBufferPool != NULL)
				error = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes( kCFAllocatorDefault, self.cvBufferPool, NULL, &outPixelBuf );

			if ((error != kCVReturnSuccess) || (outPixelBuf == NULL))
			{
				OSType pixelFormatType = CVPixelBufferGetPixelFormatType( cvImageBuffer );

				NSDictionary *pixelBufferAttributes = @{ (id) kCVPixelBufferIOSurfacePropertiesKey : @{} };

				error = CVPixelBufferCreate( nil, CVPixelBufferGetWidth( cvImageBuffer ),
											 CVPixelBufferGetHeight( cvImageBuffer ),
											 pixelFormatType,
											 (__bridge CFDictionaryRef) pixelBufferAttributes,
											 &outPixelBuf );
			}

			if ((error == kCVReturnSuccess) && (outPixelBuf != NULL))
			{
				NSSet *commonKeys = [NSSet setWithArray:(NSArray *) CMVideoFormatDescriptionGetExtensionKeysCommonWithImageBuffers()];

				NSDictionary *attachments = (NSDictionary *) CVBufferGetAttachments( cvImageBuffer, kCVAttachmentMode_ShouldPropagate );
				[attachments enumerateKeysAndObjectsUsingBlock: ^(id key, id obj, BOOL *stop)
				 {
					if ([commonKeys containsObject: key])
					{
						CVBufferSetAttachment( outPixelBuf, (__bridge CFStringRef)(key), (__bridge CFTypeRef)(obj), kCVAttachmentMode_ShouldPropagate);
					}
				}];

				attachments = (NSDictionary *) CVBufferGetAttachments( cvImageBuffer, kCVAttachmentMode_ShouldNotPropagate );

				[attachments enumerateKeysAndObjectsUsingBlock:^(id key, id obj, BOOL *stop)
				 {
					if ([commonKeys containsObject: key])
					{
						CVBufferSetAttachment( outPixelBuf, (__bridge CFStringRef)(key), (__bridge CFTypeRef)(obj), kCVAttachmentMode_ShouldNotPropagate);
					}
				}];

				CVPixelBufferLockBaseAddress( outPixelBuf, 0 );

				[self.videoCIContext render: finalCIImage toCVPixelBuffer: outPixelBuf
									 bounds: [finalCIImage extent]
								 colorSpace: self.deviceColorSpace];

				CVPixelBufferUnlockBaseAddress( outPixelBuf, 0 );
			}

			CMVideoFormatDescriptionRef videoInfo = NULL;
			OSStatus errStatus = CMVideoFormatDescriptionCreateForImageBuffer( kCFAllocatorDefault, outPixelBuf, &videoInfo );

			if (errStatus == noErr)
				error = CMSampleBufferCreateReadyWithImageBuffer( kCFAllocatorDefault, outPixelBuf, videoInfo, &sampleTiming, &newSamplelBuf );

			CVPixelBufferRelease( outPixelBuf );
			CFRelease( videoInfo );
		}
		}
	}

	return  newSamplelBuf;
}

The CIFilters are created once and reused for every sample buffer:

self.mtkCompositingFilter0 = [CIFilter filterWithName: @"CISourceOverCompositing"];
self.mtkCompositingFilter1 = [CIFilter filterWithName: @"CISourceOverCompositing"];

The program works fine on most Macs (like iMacs, MacBook Pro, M1 Mini and more), but produces black frames on some machines. One example is a Mac mini (2018) 3.2GHz Core i7 with 6 cores and 16GB RAM, and another is a client's older MacBook Air. Running the app under Xcode on the Mini didn't produce any errors or warnings in the logs.

Anyone has any pointers or ideas on how to fix this problem?

Thank you.

  • Running the app under code with the CL_PRINT_TREE environment variable set to 7, I get the following errors for every frame:

    ** OpenCL Error Notification: [CL_DEVICE_NOT_AVAILABLE] : OpenCL Error : Error: build program driver returned (-1) ** ** OpenCL Error Notification: OpenCL Warning : clBuildProgram failed: could not build program for 0xffffffff (Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz) (err:-1) **```
Add a Comment