Hello there,
I'm currently working on a Hand Tracking System. I've already placed some spheres on some joint points on the left and right hand. Now I want to access the translation/position value of these entities in the update(context: Scene) function. Now my question is, is it possible to access them via .handAnchors(), or which types of .handSkeleton.joint(name) are referencing the same entity? (E.g. is AnchorEntity(.hand(.right, location: .indexFingerTip)) the same as handSkeleton.joint(.indexFingerTip). The goal would be to access the translation of the joints where a sphere has been placed per hand and to be able to update the data every frame through the update(context) function.
I would very much appreciate any help!
See code example down below:
ImmersiveView.swift
import SwiftUI
import RealityKit
import ARKit
struct ImmersiveView: View {
public var body: some View {
RealityView { content in
/* HEAD */
let headEntity = AnchorEntity(.head)
content.add(headEntity)
/* LEFT HAND */
let leftHandWristEntity = AnchorEntity(.hand(.left, location: .wrist))
let leftHandIndexFingerEntity = AnchorEntity(.hand(.left, location: .indexFingerTip))
let leftHandWristSphere = ModelEntity(mesh: .generateSphere(radius: 0.02), materials: [SimpleMaterial(color: .red, isMetallic: false)])
let leftHandIndexFingerSphere = ModelEntity(mesh: .generateSphere(radius: 0.01), materials: [SimpleMaterial(color: .orange, isMetallic: false)])
leftHandWristEntity.addChild(leftHandWristSphere)
content.add(leftHandWristEntity)
leftHandIndexFingerEntity.addChild(leftHandIndexFingerSphere)
content.add(leftHandIndexFingerEntity)
}
}
}
TrackingSystem.swift
import SwiftUI
import simd
import ARKit
import RealityKit
public class TrackingSystem: System {
static let query = EntityQuery(where: .has(AnchoringComponent.self))
private let arKitSession = ARKitSession()
private let worldTrackingProvider = WorldTrackingProvider()
private let handTrackingProvider = HandTrackingProvider()
public required init(scene: RealityKit.Scene) {
setUpSession()
}
private func setUpSession() {
Task {
do {
try await arKitSession.run([worldTrackingProvider, handTrackingProvider])
} catch {
print("Error: \(error)")
}
}
}
public func update(context: SceneUpdateContext) {
guard worldTrackingProvider.state == .running && handTrackingProvider.state == .running else { return }
let _ = context.entities(matching: Self.query, updatingSystemWhen: .rendering)
if let avp = worldTrackingProvider.queryDeviceAnchor(atTimestamp: currentTime) {
let hands = handTrackingProvider.handAnchors(at: currentTime)
...
}
}
}