Correctly acquire and release Drawables for paused MTKView

Hi, you create a project using the built in Game template using Metal (so change the default from SceneKit), and it contains a basic example of handling the rendering loop. I think if you start from there and modify it to include the pause logic you have above you may be able to pinpoint your error.

func draw(in view: MTKView) {
    /// Per frame updates hare

    _ = inFlightSemaphore.wait(timeout: DispatchTime.distantFuture)

    if let commandBuffer = commandQueue.makeCommandBuffer() {
        let semaphore = inFlightSemaphore
        commandBuffer.addCompletedHandler { (_ commandBuffer)-> Swift.Void in
            semaphore.signal()
        }

        /// Update buffers before drawing...    
        
        /// Delay getting the currentRenderPassDescriptor until we absolutely need it to avoid
        ///   holding onto the drawable and blocking the display pipeline any longer than necessary
        let renderPassDescriptor = view.currentRenderPassDescriptor
        
        if let renderPassDescriptor = renderPassDescriptor {
            
            /// Final pass rendering code here
            if let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) {
                renderEncoder.label = "Primary Render Encoder"

                /// Render scene using render encoder            
                
                renderEncoder.endEncoding()
                
                if let drawable = view.currentDrawable {
                    commandBuffer.present(drawable)
                }
            }
        }
        
        commandBuffer.commit()
    }
}

Thank you for your response. Following your suggestion, I started with the Game template with Metal. After making the following changes to trigger draw calls on touch interactions, indicated by the git diff below, I had the same failure case as above. The first draw call functions correctly. The second call gives [CAMetalLayerDrawable texture] should not be called after already presenting this drawable. Get a nextDrawable instead. and Each CAMetalLayerDrawable can only be presented once!. So the logic to correctly iterate to the next drawable from the MTKView is missing somewhere.

@@ -17,10 +17,11 @@ class GameViewController: UIViewController {
     override func viewDidLoad() {
         super.viewDidLoad()

-        guard let mtkView = view as? MTKView else {
+        guard let newMtkView = view as? MTKView else {
             print("View of Gameview controller is not an MTKView")
             return
         }
+        mtkView = newMtkView // allow us to refer to view stored in the implicitly unwrapped optional

         // Select the device to render with.  We choose the default device
         guard let defaultDevice = MTLCreateSystemDefaultDevice() else {
@@ -31,6 +32,9 @@ class GameViewController: UIViewController {
         mtkView.device = defaultDevice
         mtkView.backgroundColor = UIColor.black

+        mtkView.isPaused = true
+        mtkView.enableSetNeedsDisplay = false
+
         guard let newRenderer = Renderer(metalKitView: mtkView) else {
             print("Renderer cannot be initialized")
             return
@@ -42,4 +46,10 @@ class GameViewController: UIViewController {

         mtkView.delegate = renderer
     }
+
+    override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
+        autoreleasepool {
+            renderer.draw(in: mtkView)
+        }
+    }
 }

I did hack together a minimal Metal app using SwiftUI with the MTKView wrapped in a UIRepresentable. Tapping the button changes the colour of the MTKView. I am quite new to iOS so beware of my likely less than optimal example. The problem is solved by grabbing the CAMetalLayer through the current drawable and using it to call nextDrawable(). Now the MTLRenderPassDescriptor must be created manually using the new drawable. This solution works but feels very uncomfortable so I can't believe it is the intended approach - grabbing so indirectly at the lower levels from MTKView (I also have yet to handle the depth and stencil render pass attachments). I would be very grateful if anyone can point me to a better solution of how to handle manual draw calls to an MTKView

Best regards,

App.swift:

import SwiftUI
import MetalKit

@main
struct ManualRenderingTestApp: App {
    var body: some Scene {
        WindowGroup { ContentView() }
    }
}
struct ContentView: View {
    let view = ColorView()
    var body: some View {
        VStack {
            view
            Button("Change color") { view.changeColor() }
        }
    }
}
class ColorViewCoordinator : NSObject, MTKViewDelegate {
    let view: MTKView
    var commandQueue: MTLCommandQueue!
    var pipelineState: MTLRenderPipelineState!
    let inFlightSemaphore = DispatchSemaphore(value: 3)

    let vertexData: [Float] = [-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0]
    var colors = [simd_float4(1.0, 0.0, 0.0, 1.0), simd_float4(0.0, 1.0, 0.0, 1.0), simd_float4(0.0, 0.0, 1.0, 1.0), simd_float4(1.0, 1.0, 0.0, 1.0)]
    var colorIndex = 0

    init(mtkView: MTKView) {
        self.view = mtkView

        let metalDevice = MTLCreateSystemDefaultDevice()!
        view.device = metalDevice
        super.init()

        self.commandQueue = metalDevice.makeCommandQueue()!
        let library = metalDevice.makeDefaultLibrary()!

        let vertexDescriptor = MTLVertexDescriptor()
        vertexDescriptor.attributes[0].format = .float2
        vertexDescriptor.attributes[0].offset = 0
        vertexDescriptor.attributes[0].bufferIndex = 0
        vertexDescriptor.layouts[0].stride = MemoryLayout<SIMD2<Float>>.stride
        vertexDescriptor.layouts[0].stepRate = 1
        vertexDescriptor.layouts[0].stepFunction = .perVertex

        let pipelineDescriptor = MTLRenderPipelineDescriptor()
        pipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm
        pipelineDescriptor.vertexFunction = library.makeFunction(name: "planeVertexShader")
        pipelineDescriptor.fragmentFunction = library.makeFunction(name: "constantColorFragmentShader")
        pipelineDescriptor.vertexDescriptor = vertexDescriptor
        do {
            pipelineState = try metalDevice.makeRenderPipelineState(descriptor: pipelineDescriptor)
        } catch {
            fatalError("Failed to create visualization render pipeline.")
        }
    }

    func incrementColor() { colorIndex = (colorIndex + 1) % colors.count }

    func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {}

    func getCALayer(view: MTKView) -> CAMetalLayer? { if let drawable = view.currentDrawable { return drawable.layer }; return nil }

    func createRenderPassDescriptor(view: MTKView, drawable: CAMetalDrawable) -> MTLRenderPassDescriptor {
        let renderPassDescriptor = MTLRenderPassDescriptor()
        renderPassDescriptor.colorAttachments[0].texture = drawable.texture
        renderPassDescriptor.colorAttachments[0].loadAction = .clear
        renderPassDescriptor.colorAttachments[0].storeAction = .store
        // depth and stencil attachments have to be set manually here
        return renderPassDescriptor
    }

    func draw(in view: MTKView) {
        _ = inFlightSemaphore.wait(timeout: DispatchTime.distantFuture)

        guard let commandBuffer = commandQueue.makeCommandBuffer() else { return }
        let semaphore = inFlightSemaphore
        commandBuffer.addCompletedHandler { (_ commandBuffer)-> Swift.Void in
            semaphore.signal()
        }
        guard let layer = getCALayer(view: view) else { return }
        guard let drawable = layer.nextDrawable() else { return }
        let renderPassDescriptor = createRenderPassDescriptor(view: view, drawable: drawable)
        guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return }
        renderEncoder.setVertexBytes(vertexData, length: vertexData.count * MemoryLayout<Float>.stride, index: 0)
        renderEncoder.setFragmentBytes(&colors[colorIndex], length: MemoryLayout<SIMD4<Float>>.stride, index: 0) // set color from user input
        renderEncoder.setRenderPipelineState(pipelineState)
        renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
        renderEncoder.endEncoding()
        commandBuffer.present(drawable)
        commandBuffer.commit()
    }
}
struct ColorView: UIViewRepresentable {
    var mtkView: MTKView = MTKView()

    func makeCoordinator() -> ColorViewCoordinator {
        return ColorViewCoordinator(mtkView: mtkView)
    }

    func makeUIView(context: UIViewRepresentableContext<ColorView>) -> MTKView {
        mtkView.delegate = context.coordinator
        mtkView.isPaused = true // only push data
        mtkView.enableSetNeedsDisplay = false // only push data
        mtkView.framebufferOnly = true // we don't render to anything but the screen
        mtkView.colorPixelFormat = .bgra8Unorm
        mtkView.drawableSize = mtkView.frame.size
        return mtkView
    }

    func updateUIView(_ uiView: MTKView, context: UIViewRepresentableContext<ColorView>) {}

    func changeColor() {
        let coordinator = mtkView.delegate as? ColorViewCoordinator
        coordinator!.incrementColor()
        autoreleasepool {
            mtkView.delegate!.draw(in: mtkView)
        }
    }
}

Shaders.Metal:

#include <metal_stdlib>
using namespace metal;

typedef struct { float2 position [[ attribute(0) ]]; } VertexIn;
typedef struct { float4 position [[position]]; } FragIn;

vertex FragIn planeVertexShader(VertexIn in [[stage_in]]) {
    FragIn out;
    out.position = float4(in.position, 0.0, 1.0);
    return out;
}

fragment float4 constantColorFragmentShader(
    FragIn in [[stage_in]],
    constant float4& color
) { return color; }