trying to put custom material on ARMeshAnchor, but during the session the FPS frequency begins to decrease, and the following warning comes out
Link to the video example of braking FPS
https://www.dropbox.com/s/p7g7qgvb5o95gdf/RPReplay_Final1637641112.mov?dl=0
Console Warning
ARSessionDelegate is retaining 11 ARFrames. This can lead to future camera frames being dropped
CameraViewController
final class CameraViewController: UIViewController {
private let arView = ARView().do {
$0.automaticallyConfigureSession = false
}
private var meshAnchorTracker: MeshAnchorTracker?
override func viewDidLoad() {
super.viewDidLoad()
MetalLibLoader.initializeMetal()
setupSubviews()
}
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
setupARSession()
}
private func setupSubviews() {
view.addSubview(arView)
arView.frame = view.bounds
}
private func setupARSession() {
configureWorldTracking()
setupPhysicsOrigin()
}
private func configureWorldTracking() {
let configuration = ARWorldTrackingConfiguration()
let sceneReconstruction: ARWorldTrackingConfiguration.SceneReconstruction = .mesh
if ARWorldTrackingConfiguration.supportsSceneReconstruction(sceneReconstruction) {
configuration.sceneReconstruction = sceneReconstruction
meshAnchorTracker = .init(arView: arView)
}
configuration.planeDetection = .horizontal
arView.session.run(configuration, options: [.resetSceneReconstruction])
arView.renderOptions.insert(.disableMotionBlur)
arView.session.delegate = self
}
private func setupPhysicsOrigin() {
let physicsOrigin = Entity()
physicsOrigin.scale = .init(repeating: 0.1)
let anchor = AnchorEntity(world: SIMD3<Float>())
anchor.addChild(physicsOrigin)
arView.scene.addAnchor(anchor)
arView.physicsOrigin = physicsOrigin
}
func updateAnchors(anchors: [ARAnchor]) {
for anchor in anchors.compactMap({ $0 as? ARMeshAnchor }) {
meshAnchorTracker?.update(anchor)
}
}
}
extension CameraViewController: ARSessionDelegate {
func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
updateAnchors(anchors: anchors)
}
func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
updateAnchors(anchors: anchors)
}
func session(_ session: ARSession, didRemove anchors: [ARAnchor]) {
for anchor in anchors.compactMap({ $0 as? ARMeshAnchor }) {
meshAnchorTracker?.remove(anchor)
}
}
}
MeshAnchorTracker
struct MeshAnchorTracker {
var entries: [ARMeshAnchor: Entry] = [:]
weak var arView: ARView?
init(arView: ARView) {
self.arView = arView
}
class Entry {
var entity: AnchorEntity
private var currentTask: Cancellable?
var nextTask: LoadRequest<MeshResource>? {
didSet {
scheduleNextTask()
}
}
static let material: RealityKit.CustomMaterial = {
let customMaterial: CustomMaterial
let surfaceShader = CustomMaterial.SurfaceShader(named: "plasma", in: MetalLibLoader.library)
do {
try customMaterial = CustomMaterial(surfaceShader: surfaceShader, lightingModel: .lit)
} catch {
fatalError(error.localizedDescription)
}
return customMaterial
}()
func scheduleNextTask() {
guard let task = nextTask else { return }
guard currentTask == nil else { return }
self.nextTask = nil
currentTask = task
.sink(
receiveCompletion: { result in
switch result {
case .failure(let error): assertionFailure("\(error)")
default: return
}
},
receiveValue: { [weak self] mesh in
self?.currentTask = nil
self?.entity.components[ModelComponent.self] = ModelComponent(
mesh: mesh,
materials: [Self.material]
)
self?.scheduleNextTask()
}
)
}
init(entity: AnchorEntity) {
self.entity = entity
}
}
mutating func update(_ anchor: ARMeshAnchor) {
let tracker: Entry = {
if let tracker = entries[anchor] { return tracker }
let entity = AnchorEntity(world: SIMD3<Float>())
let tracker = Entry(entity: entity)
entries[anchor] = tracker
arView?.scene.addAnchor(entity)
return tracker
}()
let entity = tracker.entity
do {
entity.transform = .init(matrix: anchor.transform)
let geom = anchor.geometry
var desc = MeshDescriptor()
let posValues = geom.vertices.asSIMD3(ofType: Float.self)
desc.positions = .init(posValues)
let normalValues = geom.normals.asSIMD3(ofType: Float.self)
desc.normals = .init(normalValues)
do {
desc.primitives = .polygons(
(0..<geom.faces.count).map { _ in UInt8(geom.faces.indexCountPerPrimitive) },
(0..<geom.faces.count * geom.faces.indexCountPerPrimitive).map {
geom.faces.buffer.contents()
.advanced(by: $0 * geom.faces.bytesPerIndex)
.assumingMemoryBound(to: UInt32.self).pointee
}
)
}
tracker.nextTask = MeshResource.generateAsync(from: [desc])
}
}
mutating func remove(_ anchor: ARMeshAnchor) {
if let entity = self.entries[anchor] {
entity.entity.removeFromParent()
self.entries[anchor] = nil
}
}
}
extension ARGeometrySource {
func asArray<T>(ofType: T.Type) -> [T] {
dispatchPrecondition(condition: .onQueue(.main))
assert(MemoryLayout<T>.stride == stride, "Invalid stride \(MemoryLayout<T>.stride); expected \(stride)")
return (0..<self.count).map {
buffer.contents().advanced(by: offset + stride * Int($0)).assumingMemoryBound(to: T.self).pointee
}
}
func asSIMD3<T>(ofType: T.Type) -> [SIMD3<T>] {
return asArray(ofType: (T, T, T).self).map { .init($0.0, $0.1, $0.2) }
}
}
What could the problem? 🥵
