URLSessionDownloadDelegate never called

My request call site:

        let headers = coordinator.requestHeaders(path: headersPath)
        var request = URLRequest(url: url, cachePolicy: .useProtocolCachePolicy, timeoutInterval: defaultTimeoutInterval)
        request.httpMethod = HTTPRequestMethod.get.rawValue
        request.allHTTPHeaderFields = headers
        
        let delegate = SessionDelegate(priority: priority, progressHandler: progressHandler)
        let session = URLSession(configuration: configuration, delegate: delegate, delegateQueue: .main)
        let (localUrl, response) = try await session.download(for: request)
        let data = try Data(contentsOf: localUrl)

SessionDelegate:

public class SessionDelegate: NSObject, URLSessionTaskDelegate, URLSessionDownloadDelegate {
    public typealias ProgressHandler = ((_ progress: Double) -> Void)
    
    public var metrics: URLSessionTaskMetrics?
    
    private let priority: TaskPriority
    private let progressHandler: ProgressHandler?
    
    //MARK: - Lifecycle
    init(priority: TaskPriority, progressHandler: ProgressHandler? = nil) {
        self.priority = priority
        self.progressHandler = progressHandler
    }
    
    //MARK: - URLSessionTaskDelegate
    public func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) {
        task.priority = priority.rawValue
    }
    
    public func urlSession(_ session: URLSession, task: URLSessionTask, didFinishCollecting metrics: URLSessionTaskMetrics) {
        self.metrics = metrics
    }
    
    public func urlSession(_ session: URLSession, task: URLSessionTask, didSendBodyData bytesSent: Int64, totalBytesSent: Int64, totalBytesExpectedToSend: Int64) {
        if let progressHandler {
            let progress = Progress(totalUnitCount: totalBytesExpectedToSend)
            progress.completedUnitCount = totalBytesSent
            progressHandler(progress.fractionCompleted)
        }
    }
    
    //MARK: - URLSessionDownloadDelegate
    public func urlSession(_ session: URLSession, downloadTask: URLSessionDownloadTask, didFinishDownloadingTo location: URL) { }
    
    public func urlSession(_ session: URLSession, downloadTask: URLSessionDownloadTask, didWriteData bytesWritten: Int64, totalBytesWritten: Int64, totalBytesExpectedToWrite: Int64) {
        if let progressHandler {
            let progress = Progress(totalUnitCount: totalBytesExpectedToWrite)
            progress.completedUnitCount = totalBytesWritten
            progressHandler(progress.fractionCompleted)
        }
    }
}

For some reason I get delegate callbacks from URLSessionDelegate and URLSessionTaskDelegate but never from URLSessionDownloadDelegate. I've tried assigning the delegate to the task or the session (or both), but behavior remains the same.
Not sure if related but when I try using background session configuration I crash on let (localUrl, response) = try await session.download(for: request) with the error Completion handler blocks are not supported in background sessions. Use a delegate instead. even though I use the async version of the download task, and assigning delegate to the session, it might point to the reason why none of the delegate functions is getting called (maybe?), but I have no clue why Xcode thinks I'm using completion handler.