Faced with this problem, it's not clear how to remove the action. However, you can check the inserted text. If the content includes anything other than numbers, it won't appear in the text field.
func textFieldDidChangeSelection(_ textField: UITextField) {
if let text = textField.text {
let filteredText = text.filter { $0.isNumber }
if text != filteredText {
textField.text = filteredText
}
}
}