Ray Picking Problems

Question

I've read so many answers on here about how to do Ray Picking, that I thought I had the idea of it down. But when I try to implement it in my game, I get garbage.

I'm working with LWJGL.

Here's the code:

public static Ray getPick(int mouseX, int mouseY){
    glPushMatrix();

    //Setting up the Mouse Clip
    Vector4f mouseClip = new Vector4f((float)mouseX * 2 / 960f - 1, 1 - (float)mouseY * 2 / 640f ,0 ,1);

    //Loading Matrices
    FloatBuffer modMatrix = BufferUtils.createFloatBuffer(16);
    FloatBuffer projMatrix = BufferUtils.createFloatBuffer(16);
    glGetFloat(GL_MODELVIEW_MATRIX, modMatrix);
    glGetFloat(GL_PROJECTION_MATRIX, projMatrix);

    //Assigning Matrices
    Matrix4f proj = new Matrix4f();
    Matrix4f model = new Matrix4f();
    model.load(modMatrix);
    proj.load(projMatrix);

    //Multiplying the Projection Matrix by the Model View Matrix
    Matrix4f tempView = new Matrix4f();
    Matrix4f.mul(proj, model, tempView);
    tempView.invert();

    //Getting the Camera Position in World Space.  The 4th Column of the Model View Matrix.
    model.invert();
    Point cameraPos = new Point(model.m30, model.m31, model.m32);

    //Theoretically getting the vector the Picking Ray goes
    Vector4f rayVector = new Vector4f();
    Matrix4f.transform(tempView, mouseClip, rayVector);
    rayVector.translate((float)-cameraPos.getX(),(float) -cameraPos.getY(),(float) -cameraPos.getZ(), 0f);
    rayVector.normalise();
    glPopMatrix();

    //This Basically Spits out a value that changes as the Camera moves.
    //When the Mouse moves, the values change around 0.001 points from screen edge to edge.
    System.out.format("Vector: %f %f %f%n", rayVector.x, rayVector.y, rayVector.z);

    //return new Ray(cameraPos, rayVector);
    return null;
}

I don't really know why this isn't working. I was hoping some more experienced eyes might be able to help me out.

I can get the camera position like a champ, it's the vector the rays going in that I can't seem to get right.

Thanks.

EDIT:

knight666,

I translated your code, and here's what I came up with:

public static Ray getPick(int mouseX, int mouseY){
    glPushMatrix();

    //Setting up the Mouse Clip
    Vector2f mouseClip = new Vector2f((mouseX / 960f) * 2f - 1f, (mouseY / 640f) * 2f - 1f);

    //Loading Matrices
    FloatBuffer modMatrix = BufferUtils.createFloatBuffer(16);
    FloatBuffer projMatrix = BufferUtils.createFloatBuffer(16);
    glGetFloat(GL_MODELVIEW_MATRIX, modMatrix);
    glGetFloat(GL_PROJECTION_MATRIX, projMatrix);

    //Assigning Matrices
    Matrix4f proj = new Matrix4f();
    Matrix4f model = new Matrix4f();
    model.load(modMatrix);
    proj.load(projMatrix);

    //Multiplying the Projection Matrix by the Model View Matrix
    Matrix4f tempView = new Matrix4f();
    Matrix4f.mul(proj, model, tempView);
    tempView.invert();

    //Getting the Camera Position in World Space.  The 4th Column of the Model View Matrix.
    //model.invert();
    Point cameraPos = new Point(-model.m30, -model.m31, -model.m32);

    //Getting Mouse Position in 3D Space
    Vector4f pOne = new Vector4f(mouseClip.x, mouseClip.y, -1f, 1f);
    Vector4f pTwo = new Vector4f(mouseClip.x, mouseClip.y,  1f, 1f);
    Vector4f nearPoint = new Vector4f();
    Vector4f farPoint = new Vector4f();
    Matrix4f.transform(tempView, pOne, nearPoint);
    Matrix4f.transform(tempView, pTwo, farPoint);

    Vector3f pOneNorm = new Vector3f(pOne.x / pOne.w, pOne.y / pOne.w, pOne.z / pOne.w);
    Vector3f pTwoNorm = new Vector3f(pTwo.x / pTwo.w, pTwo.y / pTwo.w, pTwo.z / pTwo.w);

    Vector3f rayVector = new Vector3f();
    Vector3f.sub(pTwoNorm, pOneNorm, rayVector);

    rayVector.normalise();

    glPopMatrix();

    return new Ray(cameraPos, rayVector);
    //return null;
}

Unfortunately, I'm getting a ray with the vector of (0,0,-2) that doesn't change when I move the mouse.

As I understood it, Matrix4f.transform(matrix, vector, newVector) is the same as newVector = matrix * vector. Unfortunately in LWJGL, I can't use the former. I'm not sure if that's the case of what's going wrong or not.

I appreciate the help!

Edit #2

Here's the current method, with Byte56's updates:

public static Ray getPick(int mouseX, int mouseY){
    float windowWidth = 960f;
    float windowHeight = 640f;

    float aspectRatio = windowWidth / windowHeight;

    //get the mouse position in screenSpace coords
    double screenSpaceX = ((float) mouseX / (windowWidth / 2) - 1.0f) * aspectRatio;
    double screenSpaceY = (1.0f - (float) mouseY / (windowHeight / 2));

    double viewRatio = Math.tan((float) Math.PI / 4.0f / 2.00f);

    screenSpaceX = screenSpaceX * viewRatio;
    screenSpaceY = screenSpaceY * viewRatio;

    float NearPlane = 1f;
    float FarPlane = 100f;

    //Find the far and near camera spaces
    Vector4f cameraSpaceNear = new Vector4f((float) (screenSpaceX * NearPlane), (float) (screenSpaceY * NearPlane), (float) (-NearPlane), 1);
    Vector4f cameraSpaceFar = new Vector4f((float) (screenSpaceX * FarPlane), (float) (screenSpaceY * FarPlane), (float) (-FarPlane), 1);


    //Unproject the 2D window into 3D to see where in 3D we're actually clicking
    //Loading Matrices
    FloatBuffer modMatrix = BufferUtils.createFloatBuffer(16);
    FloatBuffer projMatrix = BufferUtils.createFloatBuffer(16);
    glGetFloat(GL_MODELVIEW_MATRIX, modMatrix);
    glGetFloat(GL_PROJECTION_MATRIX, projMatrix);

    // Assigning Matrices
    Matrix4f proj = new Matrix4f();
    Matrix4f model = new Matrix4f();
    model.load(modMatrix);
    proj.load(projMatrix);

    // Multiplying the Projection Matrix by the Model View Matrix
    Matrix4f tempView = new Matrix4f();
    Matrix4f.mul(proj, model, tempView);
    tempView.invert();

    Vector4f worldSpaceNear = new Vector4f();
    Matrix4f.transform(tempView, cameraSpaceNear, worldSpaceNear);

    Vector4f worldSpaceFar = new Vector4f();

    Matrix4f.transform(tempView, cameraSpaceFar, worldSpaceFar);

    //calculate the ray position and direction
    Vector3f rayPosition = new Vector3f(worldSpaceNear.x, worldSpaceNear.y, worldSpaceNear.z);
    Vector3f rayDirection = new Vector3f(worldSpaceFar.x - worldSpaceNear.x, worldSpaceFar.y - worldSpaceNear.y, worldSpaceFar.z - worldSpaceNear.z);

    rayDirection.normalise();

    Point p = new Point(rayPosition.x, rayPosition.y, rayPosition.z);

    return new Ray(p, rayDirection);
}

Here is my gluPerspective method call:

FIELD_OF_VIEW = 30;
gluPerspective(FIELD_OF_VIEW, 960f / 640f, 1f, 100f);

And my glLookAt method call. The method references a custom Point Object that moves with the player. The matrix is Pushed, LookedAt, and Popped every Render. The values u, v, i, & j are determined and changed to be able to rotate the "camera" around the point. The gluLookAt Call always comes before the attempt to make a ray.:

gluLookAt(
            //Camera Set Above the Reference Point
            (float) (lookAtPoint.getX() + u), (float)(lookAtPoint.getY() + v), (float) lookAtPoint.getZ() + zZoom,
            //The Reference Point
            (float) lookAtPoint.getX(), (float) lookAtPoint.getY(), (float) lookAtPoint.getZ(),
            //Setting the Up Vector
            i, j, 0f);

If I negate the vector, it's still off. The X axis is counting what should be a movement of 1 as a movement of approx 0.03.

Answer 1

OK, so I found your problem. I don't, however, know why it's the problem.

I plugged your code into mine, everything was the same except the matrices. The matrices retrieved from OpenGL did not match the ones I was generating. So, if you would, try upgrading your camera class to generate those matrices for you.

There's a few functions you'll need:

Matrix4f result = new Matrix4f();
...
Matrix4f CreateLookAt(Vector3f position, Vector3f direction, Vector3f up) {
    Vector3f.cross(up, direction, rightVector);
    rightVector = (Vector3f) rightVector.normalise();

    result.m00 = rightVector.x;
    result.m10 = rightVector.y;
    result.m20 = rightVector.z;
    result.m30 = -(Vector3f.dot(rightVector, position));

    result.m01 = up.x;
    result.m11 = up.y;
    result.m21 = up.z;
    result.m31 = -(Vector3f.dot(up, position));

    result.m02 = direction.x;
    result.m12 = direction.y;
    result.m22 = direction.z;
    result.m32 = -(Vector3f.dot(direction, position));

    return result;
}

private Matrix4f glhFrustumf2(Matrix4f matrix, float left, float right, float bottom, float top, float znear, float zfar) {
    float temp, temp2, temp3, temp4;
    temp = 2.0f * znear;
    temp2 = right - left;
    temp3 = top - bottom;
    temp4 = zfar - znear;
    matrix.m00 = temp / temp2;
    matrix.m01 = 0.0f;
    matrix.m02 = 0.0f;
    matrix.m03 = 0.0f;
    matrix.m10 = 0.0f;
    matrix.m11 = temp / temp3;
    matrix.m12 = 0.0f;
    matrix.m13 = 0.0f;
    matrix.m20 = (right + left) / temp2;
    matrix.m21 = (top + bottom) / temp3;
    matrix.m22 = (-zfar - znear) / temp4;
    matrix.m23 = -1.0f;
    matrix.m30 = 0.0f;
    matrix.m31 = 0.0f;
    matrix.m32 = (-temp * zfar) / temp4;
    matrix.m33 = 0.0f;
    return matrix;
}

Matrix4f CreatePerspectiveFieldOfView(float fov, float aspect, float zn, float zf) {
    float ymax, xmax;
    ymax = (float) (zn * Math.tan(fov * Math.PI / 360.0));
    xmax = ymax * aspectRatio;
    Matrix4f result = new Matrix4f();
    glhFrustumf2(result, -xmax, xmax, -ymax, ymax, zn, zf);

    return result;
}

Then when updating your view and projection you'll use:

view = CreateLookAt(position, direction, upVector);
projection = CreatePerspectiveFieldOfView(ViewAngle * zoomFactor, (float) WindowX / (float) WindowY, NearPlane, FarPlane);

Then there's the picking code:

public Ray GetPickRay(int mouseX, int mouseY) {
    float windowWidth = WindowX;
    float windowHeight = WindowY;

    double screenSpaceX = (mouseX / (windowWidth / 2) - 1.0f) * aspectRatio;
    double screenSpaceY = (1.0f - mouseY / (windowHeight / 2));
    double viewRatio = Math.tan(((float) Math.PI / (180.f/ViewAngle) / 2.00f)) * zoomFactor;

    screenSpaceX = screenSpaceX * viewRatio;
    screenSpaceY = screenSpaceY * viewRatio;

    cameraSpaceNear.set((float) (screenSpaceX * NearPlane), (float) (screenSpaceY * NearPlane),
            (-NearPlane), 1);
    cameraSpaceFar.set((float) (screenSpaceX * FarPlane), (float) (screenSpaceY * FarPlane),
            (-FarPlane), 1);

    tmpView.load(view);
    invView = (Matrix4f) tmpView.invert();
    Vector4f colonySpaceNear = new Vector4f();
    Matrix4f.transform(invView, cameraSpaceNear, colonySpaceNear);

    Vector4f colonySpaceFar = new Vector4f();

    Matrix4f.transform(invView, cameraSpaceFar, colonySpaceFar);

    rayPosition.set(colonySpaceNear.x, colonySpaceNear.y, colonySpaceNear.z);
    rayDirection.set(colonySpaceFar.x - colonySpaceNear.x,
            colonySpaceFar.y - colonySpaceNear.y, colonySpaceFar.z - colonySpaceNear.z);

    rayDirection.normalise();

    return new Ray(rayPosition, rayDirection);
}

Finally, when you're drawing your scene, you'll load in your camera matrices.

glMatrixMode(GL_PROJECTION);
glLoadMatrix(UtilFunctions.MatrixToFloatBuffer(COLONY.Camera.projection));
glMatrixMode(GL_MODELVIEW);
glLoadMatrix(UtilFunctions.MatrixToFloatBuffer(COLONY.Camera.view));

Phew! That should keep you busy for a bit and hopefully solve your problem!

Answer 2

I'm not used to coding in Java, but I managed to fix it for you using code from Horde3D.

// Algorithm courtesy of Horde3D
// h3dutPickRay

public static Ray getPick(int mouseX, int mouseY)
{
    // Mouse position in NDC [-1, 1]

    Vector2f mouse_ndc = new Vector4f(
        (float)(mouseX / 960f) * 2f - 1f,  // TODO: Hardcoded viewport
        (float)(mouseY / 640f) * 2f - 1f,
    );

    // Get matrices

    FloatBuffer projMatrix = BufferUtils.createFloatBuffer(16);
    glGetFloat(GL_PROJECTION_MATRIX, projMatrix);
    Matrix4f proj = new Matrix4f();
    proj.load(projMatrix);

    FloatBuffer modMatrix = BufferUtils.createFloatBuffer(16);
    glGetFloat(GL_MODELVIEW_MATRIX, modMatrix);
    Matrix4f model = new Matrix4f();
    model.load(modMatrix);

    // Build inverse MVP

    Matrix4f inverse_mvp = new Matrix4f();
    Matrix4f.mul(proj, model, inverse_mvp);
    inverse_mvp.invert();

    // Ray starts at camera

    Vector3f ray_origin(
        -model.m30, 
        -model.m31, 
        -model.m32
    );

    // Get positions in NDC space

    Vector4f p0 = inverse_mvp * Vector4f(mouse_ndc.x, mouse_ndc.y, -1f, 1f);
    Vector4f p1 = inverse_mvp * Vector4f(mouse_ndc.x, mouse_ndc.y,  1f, 1f);

    // Normalize to 3D coordinates

    Vector3f p0_norm(
        p0.x / p0.w, 
        p0.y / p0.w, 
        p0.z / p0.w
    );

    Vector3f p1_norm(
        p1.x / p0.w, 
        p1.y / p0.w, 
        p1.z / p0.w
    );

    // Get ray direction

    Vector3f ray_direction = p1 - p0;
    // ray_direction.normalize(); // optional

    // Finish

    return new Ray(ray_origin, ray_direction);
}