I'm trying to use AVCaptureSession and AVAssetWriter to convert video and audio from an iPhone's camera and microphone into a fragmented video file in AppleHLS format.
Below is part of the code.
It seems that the capture is successful, and I have confirmed that the data received with captureOutput() can be appended to videoWriterIput and audioWriterInput using append().
When executing audioWriterInput!.append(sampleBuffer)
, sampleBuffer has the following value, and it looks like the audio data has been passed to AssetWriter.
sampleBuffer.duration : CMTime(value: 941, timescale: 44100, flags: __C.CMTimeFlags(rawValue: 1), epoch: 0)
sampleBuffer.totalSampleSize : 1882
However, the final output init.mp4
and *.m4s
do not contain Audio. (The video can be played without any problems.)
Could you please tell me any problems or hints as to why Audio is not included?
/// Capture Session
let captureSession = AVCaptureSession()
/// Capture Input
var videoDevice: AVCaptureDevice?
var audioDevice: AVCaptureDevice?
/// Configure and Start Capture Session
func startCapture() {
// Start Configuration
captureSession.beginConfiguration()
// Setup Input Video
videoDevice = self.defaultCamera(cameraSide: cameraSide)
videoDevice!.activeVideoMinFrameDuration = CMTimeMake(value: 1, timescale: 30)
let videoInput = try AVCaptureDeviceInput(device: videoDevice!) as AVCaptureDeviceInput
captureSession.addInput(videoInput)
// Setup Input Audio
audioDevice = AVCaptureDevice.default(for: AVMediaType.audio)
let audioInput = try AVCaptureDeviceInput(device: audioDevice!) as AVCaptureDeviceInput
captureSession.addInput(audioInput)
// Setup Output Video
let videoDataOutput = AVCaptureVideoDataOutput()
videoDataOutput.setSampleBufferDelegate(self, queue: recordingQueue)
videoDataOutput.alwaysDiscardsLateVideoFrames = true
videoDataOutput.videoSettings = [
kCVPixelBufferPixelFormatTypeKey: Int(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)] as [String : Any]
captureSession.addOutput(videoDataOutput)
// Setup Output Audio
let audioDataOutput = AVCaptureAudioDataOutput()
audioDataOutput.setSampleBufferDelegate(self, queue: recordingQueue)
captureSession.addOutput(audioDataOutput)
//End Configuration
captureSession.commitConfiguration()
// Start Capture
captureSession.startRunning()
}
private let assetWriter: AVAssetWriter?
private let startTimeOffset: CMTime
private var audioWriterInput: AVAssetWriterInput?
private let videoWriterInput: AVAssetWriterInput?
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
if assetWriter == nil {
// AssetWriter
assetWriter = AVAssetWriter(contentType: UTType(AVFileType.mp4.rawValue)!)
self.startTimeOffset = CMTime(value: 1, timescale: 1)
// Setup Input of Audio.
let audioCompressionSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44_100,
AVNumberOfChannelsKey: 1,
AVEncoderBitRateKey: 128_000
]
audioWriterInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioCompressionSettings)
audioWriterInput!.expectsMediaDataInRealTime = true
assetWriter.add(audioWriterInput!)
// Setup Input of Video.
let videoCompressionSettings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264
]
let videoCompressionSettings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: 1280,
AVVideoHeightKey: 720,
AVVideoCompressionPropertiesKey: [
kVTCompressionPropertyKey_AverageBitRate: 1_024_000,
kVTCompressionPropertyKey_ProfileLevel: kVTProfileLevel_H264_Baseline_AutoLevel
]
]
videoWriterInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoCompressionSettings)
videoWriterInput!.expectsMediaDataInRealTime = true
pixelBuffer = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoWriterInput!, sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)])
assetWriter.add(videoWriterInput!)
// Configure the asset writer for writing data in fragmented MPEG-4 format.
assetWriter.outputFileTypeProfile = AVFileTypeProfile.mpeg4AppleHLS
assetWriter.preferredOutputSegmentInterval = CMTime(seconds: 1.0, preferredTimescale: 1)
assetWriter.initialSegmentStartTime = startTimeOffset
assetWriter.delegate = self
// start AssetWriiter
startTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
assetWriter.startWriting()
assetWriter.startSession(atSourceTime: startTime)
}
let isVideo = output is AVCaptureVideoDataOutput
if isVideo {
if videoWriterInput.isReadyForMoreMediaData {
videoWriterInput!.append(sampleBuffer)
}
}else{
if audioWriterInput!.isReadyForMoreMediaData {
audioWriterInput!.append(sampleBuffer)
}
}
}
func assetWriter(_ writer: AVAssetWriter, didOutputSegmentData segmentData: Data, segmentType: AVAssetSegmentType, segmentReport: AVAssetSegmentReport?) {
:
:
}