RealityKit - Change Material Color or other properties in RealityView

In a RealityView, I have scene loaded from Reality Composer Pro. The entity I'm interacting with has a PhysicallyBasedMaterial with a diffuse color. I want to change that color when on long press. I can get the entity and even get a reference to the material, but I can't seem to change anything about it.

What is the best way to change the color of a material at runtime?

    var longPress: some Gesture {
        LongPressGesture(minimumDuration: 0.5)
            .targetedToAnyEntity()
            .onEnded { value in
                value.entity.position.y = value.entity.position.y + 0.01
                if var shadow = value.entity.components[GroundingShadowComponent.self] {
                    shadow.castsShadow = true
                    value.entity.components.set(shadow)
                }
                if let model = value.entity.components[ModelComponent.self] {
                    print("material", model)
                    if let mat = model.materials.first {
                        print("material", mat)
                        // I have a material here but I can't set any properties?
                        // mat.diffuseColor  does not exist
                    }
                }

            }
    }

Here is the full code

struct Lab5026: View {

    var body: some View {
        RealityView { content in

            if let root = try? await Entity(named: "GestureLab", in: realityKitContentBundle) {
                root.position = [0, -0.45, 0]

                if let subject = root.findEntity(named: "Cube") {
                    subject.components.set(HoverEffectComponent())
                    subject.components.set(GroundingShadowComponent(castsShadow: false))
                }

                content.add(root)
            }

        }
        .gesture(longPress.sequenced(before: dragGesture))

    }

    var longPress: some Gesture {
        LongPressGesture(minimumDuration: 0.5)
            .targetedToAnyEntity()
            .onEnded { value in
                value.entity.position.y = value.entity.position.y + 0.01
                if var shadow = value.entity.components[GroundingShadowComponent.self] {
                    shadow.castsShadow = true
                    value.entity.components.set(shadow)
                }
                if let model = value.entity.components[ModelComponent.self] {
                    print("material", model)
                    if let mat = model.materials.first {
                        print("material", mat)
                        // I have a material here but I can't set any properties?
                        // mat.diffuseColor  does not exist
//                        PhysicallyBasedMaterial
                    }
                }

            }
    }

    var dragGesture: some Gesture {
        DragGesture()
            .targetedToAnyEntity()
            .onChanged { value in

                let newPostion = value.convert(value.location3D, from: .global, to: value.entity.parent!)

                let limit: Float = 0.175
                value.entity.position.x = min(max(newPostion.x, -limit), limit)
                value.entity.position.z = min(max(newPostion.z, -limit), limit)

            }
            .onEnded { value in
                value.entity.position.y = value.entity.position.y - 0.01

                if var shadow = value.entity.components[GroundingShadowComponent.self] {
                    shadow.castsShadow = false
                     value.entity.components.set(shadow)
                }

            }
    }
}