I'm trying to get basic frustum culling against an AABB working, and I'm having a bit of trouble figuring out how to extract the frustum planes from my camera's transform matrix. All the example code I've found expects position, look, and up vectors, as you'd pass to gluLookAt. My camera matrix can be transformed by a parent and other bits of code, so those vectors never exist anywhere that I could use them.
I've looked at code for gluLookAt and tried to graft the inverse of that on top of the Lighthouse3D example for creating the Frustum planes. It's really close to correct, but I have objects flickering in and out of existence still, so something's not quite right. All of the related code is here, so I'm hoping this will be an easy find for somebody.
struct Frustum {
enum {
TOP,
BOTTOM,
LEFT,
RIGHT,
NEAR,
FAR
};
Frustum(hsMatrix44 cam, float znear, float zfar, float angle, float aspect) {
float nw, nh, fw, fh, tang;
tang = (float)tan(angle * 0.5);
nh = znear * tang;
nw = nh * aspect;
fh = zfar * tang;
fw = fh * angle;
hsVector3 dir,nc,fc,X,Y,Z,P,U;
hsVector3 ntl,ntr,nbl,nbr,ftl,ftr,fbl,fbr;
Z.X = -cam(2,0);
Z.Y = -cam(2,1);
Z.Z = -cam(2,2);
U.X = cam(1,0);
U.Y = cam(1,1);
U.Z = cam(1,2);
X = U.crossP(Z);
X = X * (1.f/X.magnitude());
Y = Z.crossP(X);
hsMatrix44 icam = cam.inverse();
P.X = icam(0,3);
P.Y = icam(1,3);
P.Z = icam(2,3);
nc = P - Z * znear;
fc = P - Z * zfar;
ntl = nc + Y * nh - X * nw;
ntr = nc + Y * nh + X * nw;
nbl = nc - Y * nh - X * nw;
nbr = nc - Y * nh + X * nw;
ftl = fc + Y * fh - X * fw;
ftr = fc + Y * fh + X * fw;
fbl = fc - Y * fh - X * fw;
fbr = fc - Y * fh + X * fw;
planes[TOP] = planeFromPoints(ntr, ntl, ftl);
planes[BOTTOM] = planeFromPoints(nbl, nbr, fbr);
planes[LEFT] = planeFromPoints(ntl, nbl, fbl);
planes[RIGHT] = planeFromPoints(nbr, ntr, fbr);
planes[NEAR] = planeFromPoints(ntl, ntr, nbr);
planes[FAR] = planeFromPoints(ftr, ftl, fbl);
}
bool checkBox(const hsBounds3Ext& box) {
for(int i = 0; i < 6; ++i) {
float d0, d1;
d0 = planes[i].W + planes[i].N.dotP(box.getMins());
d1 = planes[i].W + planes[i].N.dotP(box.getMaxs());
if((d0 < 0.f) && (d1 < 0.f))
return false;
}
return true;
}
hsPlane3 planes[6];
};
hsPlane3 planeFromPoints(hsVector3 a, hsVector3 b, hsVector3 c) {
hsPlane3 p;
hsVector3 tmp0, tmp1;
tmp0 = a - b;
tmp1 = c - b;
p.N = tmp1.crossP(tmp0);
p.N = p.N * (1.f/p.N.magnitude());
p.W = -(p.N.dotP(b));
return p;
}