This seems like a new bug in iOS 16.1(b5) where AVMultiCamSession outputs silent audio frames when back & front mics have been added to it. This issue is not seen in iOS 16.0.3 or earlier. I can't reproduce this issue with AVMultiCamPIP sample code so I believe I have some AVAudioSession or AVMultiCamSession configuration in my code that is causing this. Moreover, setting captureSession.usesApplicationAudioSession = true
also fixes the issue, but then I do not get the audio samples from both the microphones.
Here is the code:
public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection)
{
if let videoDataOutput = output as? AVCaptureVideoDataOutput {
processVideoSampleBuffer(sampleBuffer, fromOutput: videoDataOutput)
} else if let audioDataOutput = output as? AVCaptureAudioDataOutput {
processsAudioSampleBuffer(sampleBuffer, fromOutput: audioDataOutput)
}
}
private var lastDumpTime:TimeInterval?
private func processsAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, fromOutput audioDataOutput: AVCaptureAudioDataOutput) {
if lastDumpTime == nil {
lastDumpTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer).seconds
}
let time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer).seconds
if time - lastDumpTime! >= 1.0 {
dumpAudioSampleBuffer(sampleBuffer)
lastDumpTime = time
}
}
}
private func dumpAudioSampleBuffer(_ sampleBuffer:CMSampleBuffer) {
NSLog("Dumping audio sample buffer")
var audioBufferList = AudioBufferList(mNumberBuffers: 1,
mBuffers: AudioBuffer(mNumberChannels: 0, mDataByteSize: 0, mData: nil))
var buffer: CMBlockBuffer? = nil
CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, bufferListSizeNeededOut: nil, bufferListOut: &audioBufferList, bufferListSize: MemoryLayout.size(ofValue: audioBufferList), blockBufferAllocator: nil, blockBufferMemoryAllocator: nil, flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment), blockBufferOut: &buffer)
// Create UnsafeBufferPointer from the variable length array starting at audioBufferList.mBuffers
withUnsafePointer(to: &audioBufferList.mBuffers) { ptr in
let buffers = UnsafeBufferPointer<AudioBuffer>(start: ptr, count: Int(audioBufferList.mNumberBuffers))
for buf in buffers {
// Create UnsafeBufferPointer<Int16> from the buffer data pointer
let numSamples = Int(buf.mDataByteSize)/MemoryLayout<Int16>.stride
var samples = buf.mData!.bindMemory(to: Int16.self, capacity: numSamples)
for i in 0..<numSamples {
NSLog("Sample \(samples[i])")
}
}
}
}
And here is the output: