Camera Feed as a RealityKit Texture

Hello,

I'm developing a RealityKit based app. As part of this, I would like to have a material applied to 3d objects which is essentially contains a texture which is the live camera feed from the arsession.

I have the code below which does apply a texture of the camera feed to the box but it essentially only shows the camera snapshot at the time the app loads and doesn't update continuously.

I think the issue might be that there is some issue with how the delegate is setup and captureOutput is only called when the app loads instead of every frame. Open to any other approach or insight that gets the job done.

Thank you for the help!

class CameraTextureViewController: UIViewController { var arView: ARView! var captureSession: AVCaptureSession! var videoOutput: AVCaptureVideoDataOutput! var material: UnlitMaterial? var displayLink: CADisplayLink? var currentPixelBuffer: CVPixelBuffer?

var device: MTLDevice!
var commandQueue: MTLCommandQueue!
var context: CIContext!
var textureCache: CVMetalTextureCache!

override func viewDidLoad() {
    super.viewDidLoad()
    setupARView()
    setupCaptureSession()
    setupMetal()
    setupDisplayLink()
}

func setupARView() {
    arView = ARView(frame: view.bounds)
    arView.autoresizingMask = [.flexibleWidth, .flexibleHeight]
    view.addSubview(arView)
    
    let configuration = ARWorldTrackingConfiguration()
    configuration.planeDetection = [.horizontal, .vertical]
    arView.session.run(configuration)
    arView.session.delegate = self
}

func setupCaptureSession() {
    captureSession = AVCaptureSession()
    captureSession.beginConfiguration()
    
    guard let videoDevice = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back),
          let videoDeviceInput = try? AVCaptureDeviceInput(device: videoDevice),
          captureSession.canAddInput(videoDeviceInput) else { return }
    
    captureSession.addInput(videoDeviceInput)
    
    videoOutput = AVCaptureVideoDataOutput()
    videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "cameraQueue"))
    videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
    
    guard captureSession.canAddOutput(videoOutput) else { return }
    
    captureSession.addOutput(videoOutput)
    captureSession.commitConfiguration()
    
    DispatchQueue.global(qos: .userInitiated).async { [weak self] in
        self?.captureSession.startRunning()
    }
}

func setupMetal() {
    device = MTLCreateSystemDefaultDevice()
    commandQueue = device.makeCommandQueue()
    context = CIContext(mtlDevice: device)
    CVMetalTextureCacheCreate(nil, nil, device, nil, &textureCache)
}

func setupDisplayLink() {
    displayLink = CADisplayLink(target: self, selector: #selector(updateFrame))
    displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: 60, maximum: 60, preferred: 60)
    displayLink?.add(to: .main, forMode: .default)
}

@objc func updateFrame() {
    guard let pixelBuffer = currentPixelBuffer else { return }
    updateMaterial(with: pixelBuffer)
}

func updateMaterial(with pixelBuffer: CVPixelBuffer) {
    let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
    var tempPixelBuffer: CVPixelBuffer?
    let attrs = [
        kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue,
        kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue,
        kCVPixelBufferMetalCompatibilityKey: kCFBooleanTrue
    ] as CFDictionary
    CVPixelBufferCreate(kCFAllocatorDefault, CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer), kCVPixelFormatType_32BGRA, attrs, &tempPixelBuffer)
    
    guard let tempPixelBuffer = tempPixelBuffer else { return }
    context.render(ciImage, to: tempPixelBuffer)

    var textureRef: CVMetalTexture?
    let width = CVPixelBufferGetWidth(tempPixelBuffer)
    let height = CVPixelBufferGetHeight(tempPixelBuffer)
    CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, tempPixelBuffer, nil, .bgra8Unorm, width, height, 0, &textureRef)
    guard let metalTexture = CVMetalTextureGetTexture(textureRef!) else { return }

    let ciImageFromTexture = CIImage(mtlTexture: metalTexture, options: nil)!
    guard let cgImage = context.createCGImage(ciImageFromTexture, from: ciImageFromTexture.extent) else { return }
    guard let textureResource = try? TextureResource.generate(from: cgImage, options: .init(semantic: .color)) else { return }

    if material == nil {
        material = UnlitMaterial()
    }
    material?.baseColor = .texture(textureResource)

    guard let modelEntity = arView.scene.anchors.first?.children.first as? ModelEntity else {
        let mesh = MeshResource.generateBox(size: 0.2)
        let modelEntity = ModelEntity(mesh: mesh, materials: [material!])
        let anchor = AnchorEntity(world: [0, 0, -0.5])
        anchor.addChild(modelEntity)
        arView.scene.anchors.append(anchor)
        return
    }

    modelEntity.model?.materials = [material!]
}

}

extension CameraTextureViewController: AVCaptureVideoDataOutputSampleBufferDelegate { func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } currentPixelBuffer = pixelBuffer } }

extension CameraTextureViewController: ARSessionDelegate { func session(_ session: ARSession, didUpdate frame: ARFrame) { // Handle AR frame updates if necessary } }