I'm still trying to understand how to correctly convert 3D coordinates to 2D screen coordinates using convert(position:from:) and project(_:) Below is the example ContentView.swift from the default Augmented Reality App project, with a few important modifications. Two buttons have been added, one that toggles visibility of red circular markers on the screen, and a second button that adds blue spheres to the scene. Additionally a timer has been added to trigger regular screen updates. When run, the markers should line up with the spheres on screen and follow them on screen, as the camera is moved around. However, the red circles are all very far from their corresponding spheres on screen. What am I doing wrong in my conversion that is causing the circles to not line up with the spheres? // ContentView.swift import SwiftUI import RealityKit class Coordinator { var arView: ARView? var anchor: AnchorEntity? var objects: [Entity] = [] } struct ContentView : View { let timer = Timer.publish(every: 1.0/30.0, on: .main, in: .common).autoconnect() var coord = Coordinator() @State var showMarkers = false @State var circleColor: Color = .red var body: some View { ZStack { ARViewContainer(coordinator: coord).edgesIgnoringSafeArea(.all) if showMarkers { // Add circles to the screen ForEach(coord.objects) { obj in Circle() .offset(projectedPosition(of: obj)) .frame(width: 10.0, height: 10.0) .foregroundColor(circleColor) } } VStack { Button(action: { showMarkers = !showMarkers }, label: { Text(showMarkers ? "Hide Markers" : "Show Markers") }) Spacer() Button(action: { addSphere() }, label: { Text("Add Sphere") }) } }.onReceive(timer, perform: { _ in // silly hack to force circles to redraw if circleColor == .red { circleColor = Color(#colorLiteral(red: 1, green: 0, blue: 0, alpha: 1)) } else { circleColor = .red } }) } func addSphere() { guard let anchor = coord.anchor else { return } // pick random point for new sphere let pos = SIMD3<Float>.random(in: 0...0.5) print("Adding sphere at \(pos)") // Create a sphere let mesh = MeshResource.generateSphere(radius: 0.01) let material = SimpleMaterial(color: .blue, roughness: 0.15, isMetallic: true) let model = ModelEntity(mesh: mesh, materials: [material]) model.setPosition(pos, relativeTo: anchor) anchor.addChild(model) // record sphere for later use coord.objects.append(model) } func projectedPosition(of object: Entity) -> CGPoint { // convert position of object into "world space" // (i.e., "the 3D world coordinate system of the scene") // https://developer.apple.com/documentation/realitykit/entity/convert(position:to:) let worldCoordinate = object.convert(position: object.position, to: nil) // project worldCoordinate into "the 2D pixel coordinate system of the view" // https://developer.apple.com/documentation/realitykit/arview/project(_:) guard let arView = coord.arView else { return CGPoint(x: -1, y: -1) } guard let screenPos = arView.project(worldCoordinate) else { return CGPoint(x: -1, y: -1) } // At this point, screenPos should be the screen coordinate of the object's positions on the screen. print("3D position \(object.position) mapped to \(screenPos) on screen.") return screenPos } } struct ARViewContainer: UIViewRepresentable { var coordinator: Coordinator func makeUIView(context: Context) -> ARView { let arView = ARView(frame: .zero) // Create a sphere model let mesh = MeshResource.generateSphere(radius: 0.01) let material = SimpleMaterial(color: .gray, roughness: 0.15, isMetallic: true) let model = ModelEntity(mesh: mesh, materials: [material]) // Create horizontal plane anchor for the content let anchor = AnchorEntity(.plane(.horizontal, classification: .any, minimumBounds: SIMD2<Float>(0.2, 0.2))) anchor.children.append(model) // Record values needed elsewhere coordinator.arView = arView coordinator.anchor = anchor coordinator.objects.append(model) // Add the horizontal plane anchor to the scene arView.scene.anchors.append(anchor) return arView } func updateUIView(_ uiView: ARView, context: Context) {} } #Preview { ContentView() }

I'm trying to understand how to use the project(_:) function provided by ARView to convert 3D model coordinates to 2D screen coordinates, but am getting unexpected results. Below is the default Augmented Reality App project, modified to have a single button that when tapped will place a circle over the center of the provided cube. However, when the button is pressed, the circle's position does not line up with the cube. I've looked at the documentation for project(_:), but it doesn't give any details about how to convert a point from model coordinates to "the 3D world coordinate system of the scene". Is there better documentation somewhere on how to do this conversion? // ContentView.swift import SwiftUI import RealityKit class Coordinator { var arView: ARView? var anchor: AnchorEntity? var model: Entity? } struct ContentView : View { @State var coord = Coordinator() @State var circlePos = CGPoint(x: -100, y: -100) var body: some View { ZStack { ARViewContainer(coord: coord).edgesIgnoringSafeArea(.all) VStack { Spacer() Circle() .frame(width: 10, height: 10) .foregroundColor(.red) .position(circlePos) Button(action: { showMarker() }, label: { Text("Place Marker") }) } } } func showMarker() { guard let arView = coord.arView else { return } guard let model = coord.model else { return } guard let anchor = coord.anchor else { return } print("Model position is: \(model.position)") // convert position into anchor's space let modelPos = model.convert(position: model.position, to: anchor) print("Converted position is: \(modelPos)") // convert model locations to screen coordinates circlePos = arView.project(modelPos) ?? CGPoint(x: -1, y: -1) print("circle position is now \(circlePos)") } } struct ARViewContainer: UIViewRepresentable { var coord: Coordinator func makeUIView(context: Context) -> ARView { let arView = ARView(frame: .zero) coord.arView = arView // Create a cube model let mesh = MeshResource.generateBox(size: 0.1, cornerRadius: 0.005) let material = SimpleMaterial(color: .gray, roughness: 0.15, isMetallic: true) let model = ModelEntity(mesh: mesh, materials: [material]) coord.model = model // Create horizontal plane anchor for the content let anchor = AnchorEntity(.plane(.horizontal, classification: .any, minimumBounds: SIMD2<Float>(0.2, 0.2))) anchor.children.append(model) coord.anchor = anchor // Add the horizontal plane anchor to the scene arView.scene.anchors.append(anchor) return arView } func updateUIView(_ uiView: ARView, context: Context) {} } #Preview { ContentView(coord: Coordinator()) }