This function takes in mouse coordinates and picks the model from the 3d scene by shooting a ray through it. Btw the screen size is 2x2 stretching from -1 to +1 on both the x and y axis. You'll see both those in "Compute picking ray in view space"
void Pick(int sx, int sy)
{
float dist = MathHelper::Infinity;
float tmin = 0.0f;
XMMATRIX P = cam->GetProj();
// Compute picking ray in view space.
float vx = (+2.0f*sx / mClientWidth - 1.0f) / P(0, 0);
float vy = (-2.0f*sy / mClientHeight + 1.0f) / P(1, 1);
// Ray definition in view space.
XMVECTOR rayOrigin = XMVectorSet(0.0f, 0.0f, 0.0f, 1.0f);
XMVECTOR rayDir = XMVectorSet(vx, vy, 1.0f, 0.0f);
// Tranform ray to local space of Mesh.
XMMATRIX V = cam->GetView();
XMMATRIX invView = XMMatrixInverse(&XMMatrixDeterminant(V), V);
// Make the ray direction unit length for the intersection tests.
rayDir = XMVector3Normalize(rayDir);
XMMATRIX W = XMMatrixIdentity();
XMMATRIX invWorld = XMMatrixInverse(&XMMatrixDeterminant(W), W);
XMMATRIX toLocal = XMMatrixMultiply(invView, invWorld);
rayOrigin = XMVector3TransformCoord(rayOrigin, toLocal);
rayDir = XMVector3TransformNormal(rayDir, toLocal);
if (XNA::IntersectRayOrientedBox(rayOrigin, rayDir, scene["Main_Scene"]->instance[i].boundBox, &tmin))
if (tmin < dist)
{
// The model of Main_Scene and instance i was hit.
dist = tmin;
tmin = MathHelper::Infinity;
}
}