After
"Data ingestion is complete. Beginning processing..."
I get a few of these:
[espresso] ANE Batch: 1 of the async requests being waited for returned errors
[espresso] ANE Batch: Async request 1 returned error: code=5 err=Error Domain=com.apple.appleneuralengine Code=5 "processRequest:model:qos:qIndex:modelStringID:options:error:: 0xd: Program Inference overflow" UserInfo={NSLocalizedDescription=processRequest:model:qos:qIndex:modelStringID:options:error:: 0xd: Program Inference overflow}
[espresso] [Espresso::overflow_error] :0
messages, but then it goes without any messages for a few seconds.
After about 0.30 of progress, I get this:
ERROR cv3dapi.pg: Internal codes (3): 2504 3501 4011
and I get the usual
[Photogrammetry] No SfM map found in native output!
[Photogrammetry] Reconstruction failure for modelFile bla bla bla
I find very frustrating that there seem to be no documentation whatsoever for those error messages! I wish apple would document they API's better.
Anyhow, I'm using this snippet of code that I've found on a github gist to convert NSImage objects to CVPixelBuffer, booth for the color image and for the depth image:
extension NSImage {
func pixelBuffer() -> CVPixelBuffer? {
let width = self.size.width
let height = self.size.height
let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue,
kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
var pixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(kCFAllocatorDefault,
Int(width),
Int(height),
kCVPixelFormatType_32ARGB,
attrs,
&pixelBuffer)
guard let resultPixelBuffer = pixelBuffer, status == kCVReturnSuccess else {
return nil
}
CVPixelBufferLockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let pixelData = CVPixelBufferGetBaseAddress(resultPixelBuffer)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(data: pixelData,
width: Int(width),
height: Int(height),
bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(resultPixelBuffer),
space: rgbColorSpace,
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue) else {return nil}
// context.translateBy(x: 0, y: height)
// context.scaleBy(x: 1.0, y: -1.0)
let graphicsContext = NSGraphicsContext(cgContext: context, flipped: false)
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = graphicsContext
draw(in: CGRect(x: 0, y: 0, width: width, height: height))
NSGraphicsContext.restoreGraphicsState()
CVPixelBufferUnlockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
return resultPixelBuffer
}
func depthPixelBuffer() -> CVPixelBuffer? {
let width = self.size.width
let height = self.size.height
let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue,
kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
var pixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(kCFAllocatorDefault,
Int(width),
Int(height),
kCVPixelFormatType_DepthFloat32,
attrs,
&pixelBuffer)
guard let resultPixelBuffer = pixelBuffer, status == kCVReturnSuccess else {
return nil
}
CVPixelBufferLockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let pixelData = CVPixelBufferGetBaseAddress(resultPixelBuffer)
// let linearGraySpace = CGColorSpace(name: CGColorSpace.linearGray)
let linearGraySpace = CGColorSpaceCreateDeviceGray()
guard let context = CGContext(data: pixelData,
width: Int(width),
height: Int(height),
bitsPerComponent: 32,
bytesPerRow: CVPixelBufferGetBytesPerRow(resultPixelBuffer),
space: linearGraySpace,
bitmapInfo: CGImageAlphaInfo.none.rawValue | CGBitmapInfo.floatComponents.rawValue)
else {
return nil
}
let graphicsContext = NSGraphicsContext(cgContext: context, flipped: false)
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = graphicsContext
draw(in: CGRect(x: 0, y: 0, width: width, height: height))
NSGraphicsContext.restoreGraphicsState()
CVPixelBufferUnlockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
return resultPixelBuffer
}
}
The original depth is the TIF image (greyscale floating point) saved out by the TakingPicturesFor3DObjectCapture
sample, together with the gravity.txt for each picture which I'm also reading back in the photogrammetrySample.