How to change SwiftUI TextField FocusState without having View refresh and cause a bounce effect

I have a bunch of Textfields and I want to have the user be able to hit Enter/Next on the keyboard to go through the textfields which are not in a form. They are in multiple views and one is a TextArea which makes certain things not work.

I made an example but it seems to refresh the view when the user goes from one textfield to the next, except for at the end.

I have this code, please try it out or read it, any suggestions appreciated.

import SwiftUI

class MyObject: Hashable, Equatable, ObservableObject {
    public let name: String
    @Published public var value: String
    init(name: String, value: String) {
        self.name = name
        self.value = value
    }

    static func == (lhs: MyObject, rhs: MyObject) -> Bool {
        return lhs.name == rhs.name
    }

    func hash(into hasher: inout Hasher) {
        hasher.combine(name);
        hasher.combine(value);
    }
}

class MyObjViewModel: ObservableObject {

    @Published var myObjects: [MyObject] = []
    @Published var focus: MyObject?
    
    func nextFocus() {
        guard let focus = focus,
              let index = self.myObjects.firstIndex(of: focus) else {
            return
        }
        self.focus = myObjects.indices.contains(index + 1) ? myObjects[index + 1] : nil
    }
}

struct ContentView: View {

    @ObservedObject var viewModel = MyObjViewModel()

    init() {
        self.viewModel.myObjects.append(contentsOf:[
            MyObject(name: "aa", value: "1"),
            MyObject(name: "bb", value: "2"),
            MyObject(name: "cc", value: "3"),
            MyObject(name: "dd", value: "4")
        ])
    }

    var body: some View {
        VStack {
            ForEach(self.viewModel.myObjects, id: \.self) { obj in
                FocusField(viewModel: viewModel, displayObject: obj)
            }
        }
    }
}

struct FocusField: View {

    @ObservedObject var viewModel: MyObjViewModel
    @ObservedObject var displayObject: MyObject
    @FocusState var isFocused: Bool

    var body: some View {
        TextField("Test", text: $displayObject.value)
            .onChange(of: viewModel.focus, perform: { newValue in
                self.isFocused = newValue == displayObject
            })
            .focused(self.$isFocused)
            .submitLabel(.next)
            .onSubmit {
                if self.viewModel.focus == nil {
                    self.viewModel.focus = self.displayObject
                }
                print(displayObject.name)
                self.viewModel.nextFocus()
            }
    }
}