Unexpected "asymmetrical" animation hitch in Carousel, why?

I'm trying to create a simple carousel with infinite circular scroll (that is, swiping on the last image leads to the first one and the other way around). I came up with the following solution but unfortunately, only when scrolling forward, the animation hitches before showing the correct new image.

When scrolling backwards, the animation is fine, which confuses me since the logic for backward and forward swipe is symmetrical and shares most of the code.

Here is an MRE:

I used 5 1920x1080 images named "PH1", "PH2", ... , "PH5" for the sake of this example.

CarouselModel.swift

class CarouselModel: ObservableObject {
    var uuid: UUID = UUID()

    @Published internal var onScreenBuffer: [BufferedImage] = []
    private var offScreenBuffer: [BufferedImage] = []
    internal var currentImage: Int = 0
    internal var offsetImageSelector: Int = 0

    @Published internal var imagesIDs: [String] = [] {
    didSet {
            guard imagesIDs.count > 0 else { return }
            self.makeBuffer(for: self.currentImage, buffer: &self.onScreenBuffer)
            self.makeBuffer(for: self.currentImage, buffer: &self.offScreenBuffer)
        }
    }

    internal var bufferedImagesIDs: [BufferedImage] {
        return self.imagesIDs.enumerated().map { i, image in
            return BufferedImage(assetImageName: image, position: i)
        }
    }

    init(imagesIDs: [String]) {
        self.imagesIDs = imagesIDs
    }

    internal func makeBuffer(for centralIdx: Int, buffer: inout [BufferedImage])  {
        precondition(centralIdx >= 0 && centralIdx < self.imagesIDs.count)
        var tempBuffer: [BufferedImage] = [BufferedImage].init(repeating: .zero, count: 3)
    
        for i in -1...1 {
            let next = (centralIdx + i + imagesIDs.count) % imagesIDs.count
            tempBuffer[i+1] = BufferedImage(assetImageName: imagesIDs[next], position: i+1)
        }
    
        buffer = tempBuffer
    }

    func prepareForNext(forward: Bool = true) {
        let nextIdx = forward ? (currentImage+1)%imagesIDs.count : (currentImage-1+imagesIDs.count) % imagesIDs.count
    
        if forward {
            self.offsetImageSelector -= 1
        } else {
            self.offsetImageSelector += 1
        }
    
        assert(nextIdx >= 0 && nextIdx <= imagesIDs.count)
    
        self.makeBuffer(for: nextIdx, buffer: &self.offScreenBuffer)
        currentImage = nextIdx
    }

    func swapOnscreenBuffer() {
        self.onScreenBuffer = Array(self.offScreenBuffer)
        self.offsetImageSelector = 0
    }


    internal struct BufferedImage: Identifiable {
        var assetImageName: String
        var position: Int
        var id: String
    
        init(assetImageName: String, position: Int) {
            self.assetImageName = assetImageName
            self.position = position
            self.id = "\(assetImageName)\(position)"
        }
    
        public static let zero = BufferedImage(assetImageName: "placeholder", position: .zero)
    }

}

ContentView.swift

import SwiftUI

struct ContentView: View {

    @StateObject private var carouselModel = CarouselModel(
        imagesIDs: (1..<6).map { i in
            return "PH\(i)"
        }
    )
    @State private var dragOffset: CGFloat = .zero

    var body: some View {
        GeometryReader { geo in
            VStack {
                HStack(spacing: 0) {
                    ForEach(carouselModel.onScreenBuffer, id: \.id) { bufferedImage in
                        Image(bufferedImage.assetImageName)
                            .resizable()
                            .aspectRatio(16.0/9.0, contentMode: .fit)
                            .frame(width: geo.size.width + geo.safeAreaInsets.leading + geo.safeAreaInsets.trailing, alignment: .leading)
                    }
                }
                .offset(
                    x: -(geo.size.width + geo.safeAreaInsets.leading + geo.safeAreaInsets.trailing - dragOffset)*CGFloat((1-carouselModel.offsetImageSelector))
                )
            
                .frame(maxWidth: geo.size.width + geo.safeAreaInsets.leading + geo.safeAreaInsets.trailing, alignment: .leading)
                .clipped()
                .overlay {
                    ChevronButtonsOverlay(galleryWidth: geo.size.width + geo.safeAreaInsets.leading + geo.safeAreaInsets.trailing)
                }
            
            }
        }
    }

    @ViewBuilder
    func ChevronButton() -> some View {
        ZStack {
            Circle()
                .fill(.ultraThinMaterial)
                .frame(width: 35, height: 35)
                .background {
                    Color(UIColor.label).opacity(0.8).clipShape(Circle())
                }
        
            Image(systemName: "chevron.left")
                .font(.system(size: 15).weight(.black))
                .blendMode(.destinationOut)
                .offset(y: -1)
        }
        .compositingGroup()
    }

    @ViewBuilder
    func ChevronButtonsOverlay(galleryWidth: CGFloat) -> some View {
        HStack(alignment: .center, spacing: 0) {
            Button(action: {
                if abs(carouselModel.offsetImageSelector) < 1 {
                    withAnimation(.easeOut(duration: 0.25)) {
                        carouselModel.prepareForNext(forward: false)
                        self.dragOffset = galleryWidth
                    }
                
                    DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) {
                        self.dragOffset = 0
                        carouselModel.swapOnscreenBuffer()
                    }
                }
            }) {
                ChevronButton()
            }
        
            Spacer()
        
            Button(action: {
                if abs(carouselModel.offsetImageSelector) < 1 {
                    withAnimation(.easeOut(duration: 0.25)) {
                        carouselModel.prepareForNext(forward: true)
                        self.dragOffset = -galleryWidth
                    }
                
                    DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) {
                        self.dragOffset = 0
                        carouselModel.swapOnscreenBuffer()
                    }
                }
            }) {
                ChevronButton()
            }
            .rotationEffect(.degrees(180))
        }
        .padding(5)
        .frame(maxWidth: .infinity, maxHeight: .infinity)
    }
}