After a day and a half and a couple of missteps I was able to get rid of my call to the depreciated gluPickMatrix(). I did it by bracketing the Z axis with clip planes and then rotating the sight line to the mouse position onto the -Z axis:
-(NSInteger)pick: (NSPoint)point
{
GLuint selected[20][4],
min,
hits;
NSInteger glPickName = -1;
int n;
GLint viewport[4];
GLfloat unit, d , rho , delta;
GLKVector3 zAxis, sightLine , rotAxis;
GLdouble clipPlane[4][4] =
{
{ -1.0 , 0.0 , 0.0 , 1.0 },
{ 1.0 , 0.0 , 0.0 , 1.0 },
{ 0.0 , -1.0 , 0.0 , 1.0 },
{ 0.0 , 1.0 , 0.0 , 1.0 }
};
[[self openGLContext] makeCurrentContext];
glGetIntegerv(GL_VIEWPORT , viewport );
// make the hit point relative to the center of the view
point.x = point.x - viewport[2] / 2.0;
point.y = point.y - viewport[3] / 2.0;
// Normalize the hit point:
// My frustum is set up such that
// distance 1.0 in model space == distance to the nearest edge of the view rectangle
if(viewport[2] < viewport[3])
unit = viewport[2] / 2.0;
else
unit = viewport[3] / 2.0;
point.x /= unit;
point.y /= unit;
delta = 1.0 / unit; // one pixle in model space
for( n = 0 ; n < 4 ; n++ )
clipPlane[n][3] *= delta;
//Calculate the rotation necessary to place the hit point onto the Z axis
zAxis = GLKVector3Make( 0.0 , 0.0 , -NEAR_PLANE );
sightLine = GLKVector3Make( point.x , point.y , -NEAR_PLANE );
rotAxis = GLKVector3CrossProduct( sightLine , zAxis );
d = sqrtf( point.x * point.x + point.y * point.y + NEAR_PLANE * NEAR_PLANE);
rho = acosf( NEAR_PLANE / d );
rho = 180.0 * rho / M_PI;
glMatrixMode( GL_MODELVIEW );
glPushMatrix();
glLoadIdentity();
// Clip everything more than 1 pixel from the Z axis
glEnable( GL_CLIP_PLANE0 );
glEnable( GL_CLIP_PLANE1 );
glEnable( GL_CLIP_PLANE2 );
glEnable( GL_CLIP_PLANE3 );
glClipPlane(GL_CLIP_PLANE0, clipPlane[0] );
glClipPlane(GL_CLIP_PLANE1, clipPlane[1] );
glClipPlane(GL_CLIP_PLANE2, clipPlane[2] );
glClipPlane(GL_CLIP_PLANE3, clipPlane[3] );
// Rotate the hit point onto the Z axis
glRotatef( rho , rotAxis.v[0] , rotAxis.v[1] , rotAxis.v[2] );
glSelectBuffer( sizeof(GLuint [20][4]) , (GLuint *)selected );
glRenderMode(GL_SELECT);
glInitNames();
glPushName(0);
// Notify my model objects to draw themselves
[self position];
[[NSNotificationCenter defaultCenter] postNotificationName: VN_RENDER
object: self ];
glFlush();
hits = glRenderMode( GL_RENDER );
min = UINT32_MAX;
for( n = 0 ; n < hits ; n++)
{
if( selected[n][1] < min )
{
min = selected[n][1];
glPickName = selected[n][3];
}
}
glDisable( GL_CLIP_PLANE0 );
glDisable( GL_CLIP_PLANE1 );
glDisable( GL_CLIP_PLANE2 );
glDisable( GL_CLIP_PLANE3 );
glPopMatrix();
return glPickName;
}