Rotating camera in 3D without yawing diagonally

I'm programming a C raytracer. I'm having trouble getting the camera to rotate the way I intend.

I would like the camera to rotate by pressing keys:

• F & H rotate left & right around the world's vertical axis (Y)

• T & G rotate up & down around an axis that is always horizontal and perpendicular to the view axis.

• R & Y rotate around the camera's viewing axis (local Z)

But with my current code, rotating up and then left rotates on a diagonal, instead of going around the world Y axis:

How can I make this rotation control more intuitive?

This is how the camera axes are initialized at the beginning of the program (I do this once) :

void    init_camera_vecs(t_camera *cam)
{
t_vec3  forward;

forward = get_normalised(invert(cam->vec));
cam->up = (t_vec3){0, 0, 0};
if (fabs(cam->vec.y) > 0.7)
cam->right = vec_cross((t_vec3){0, 0, -1}, forward);
else
cam->right = vec_cross((t_vec3){0, 1, 0}, forward);
cam->up = vec_cross(forward, cam->right);
}


This is my get_ray_dir() function which gives the direction of the ray from camera space to world space:

static t_vec3   lookat(t_camera *cam, t_vec3 ray_coord)
{
t_vec3  forward;
t_vec3  ray_dir;

forward = get_normalised(invert(cam->vec));
ray_dir.x = ray_coord.x * cam->right.x + ray_coord.y * cam->up.x
+ ray_coord.z * forward.x;
ray_dir.y = ray_coord.x * cam->right.y + ray_coord.y * cam->up.y
+ ray_coord.z * forward.y;
ray_dir.z = ray_coord.x * cam->right.z + ray_coord.y * cam->up.z
+ ray_coord.z * forward.z;
return (ray_dir);
}

static t_vec3   get_ray_dir(int i, int j, t_resolution res, t_camera *cam)
{
t_ray ray;
float scale;
float point_x;
float point_y;
float ratio;

ratio = res.x / (float)res.y;
scale = tan(cam->fov / 2 * M_PI / 180);
point_x = (2 * (j + 0.5) / (float)res.x - 1) * ratio * scale;
point_y = (1 - 2 * (i + 0.5) / res.y) * scale;
ray.coord = (t_vec3){point_x, point_y, -1};
ray.dir = lookat(cam, ray.coord);
normalize(&ray.dir);
return (ray.dir);
}


I tried this :

void    make_rotation_x(t_vec3 *rot_x, float angle)
{
float   c;
float   s;

c = cos(angle);
s = sin(angle);
rot_x[0] = (t_vec3){1, 0, 0};
rot_x[1] = (t_vec3){0, c, -s};
rot_x[2] = (t_vec3){0, s, c};
}

void    make_rotation_y(t_vec3 *rot_y, float angle)
{
float   c;
float   s;

c = cos(angle);
s = sin(angle);
rot_y[0] = (t_vec3){c, 0, s};
rot_y[1] = (t_vec3){0, 1, 0};
rot_y[2] = (t_vec3){-s, 0, c};
}

void    make_rotation_z(t_vec3 *rot_z, float angle)
{
float   c;
float   s;

c = cos(angle);
s = sin(angle);
rot_z[0] = (t_vec3){c, -s, 0};
rot_z[1] = (t_vec3){s, c, 0};
rot_z[2] = (t_vec3){0, 0, 1};
}

void    mult_matrix(t_vec3 *res, t_vec3 *lhs, t_vec3 *rhs)
{
res[0] = rotate_vector(lhs, rhs[0]);
res[1] = rotate_vector(lhs, rhs[1]);
res[2] = rotate_vector(lhs, rhs[2]);
}

void    cpy_matrix(t_vec3 *dst, t_vec3 *src)
{
dst[0] = src[0];
dst[1] = src[1];
dst[2] = src[2];
}

void    apply_rotation_x(t_vec3 *orientation, float angle)
{
t_vec3  matrix[3];
t_vec3  res[3];

make_rotation_x(matrix, angle);
mult_matrix(res, orientation, matrix);
cpy_matrix(orientation, res);
}

void    apply_rotation_y(t_vec3 *orientation, float angle)
{
t_vec3  matrix[3];
t_vec3  res[3];

make_rotation_y(matrix, angle);
mult_matrix(res, orientation, matrix);
cpy_matrix(orientation, res);
}

void    apply_rotation_z(t_vec3 *orientation, float angle)
{
t_vec3  matrix[3];
t_vec3  res[3];

make_rotation_z(matrix, angle);
mult_matrix(res, orientation, matrix);
cpy_matrix(orientation, res);
}

void        move_cam(t_specs *s, int key)
{
t_camera    *cam;
t_vec3      orientation[3];
float       angle;

angle = 4.5 * (M_PI / 180);
cam = s->current_cam;
orientation[0] = cam->right;
orientation[1] = cam->up;
orientation[2] = cam->vec;
if (key == KEY_W)
else if (key == KEY_S)
cam->coord = vec_sub(cam->coord, cam->up);
else if (key == KEY_A)
cam->coord = vec_sub(cam->coord, cam->right);
else if (key == KEY_D)
else if (key == KEY_Q)
cam->coord = vec_sub(cam->coord, cam->vec);
else if (key == KEY_E)
else if (key == KEY_H)
apply_rotation_y(orientation, -angle);
else if (key == KEY_F)
apply_rotation_y(orientation, angle);
else if (key == KEY_G)
apply_rotation_x(orientation, -angle);
else if (key == KEY_T)
apply_rotation_x(orientation, angle);
else if (key == KEY_R)
apply_rotation_z(orientation, -angle);
else if (key == KEY_Y)
apply_rotation_z(orientation, angle);
s->current_cam->right = orientation[0];
s->current_cam->up = orientation[1];
s->current_cam->vec = orientation[2];
normalize(&s->current_cam->right);
normalize(&s->current_cam->up);
normalize(&s->current_cam->vec);
}


I use this function now for rotation.

void        move_cam(t_specs *s, int key)
{
t_camera    *cam;
t_vec3      orientation[3];

cam = s->current_cam;
orientation[0] = cam->right;
orientation[1] = cam->up;
orientation[2] = cam->forward;

if (key == KEY_W)
else if (key == KEY_S)
cam->coord = vec_sub(cam->coord, cam->up);
else if (key == KEY_A)
cam->coord = vec_sub(cam->coord, cam->right);
else if (key == KEY_D)
else if (key == KEY_Q)
cam->coord = vec_sub(cam->coord, cam->vec);
else if (key == KEY_E)
else if (key == KEY_H)
cam->yaw += 0.1;
else if (key == KEY_F)
cam->yaw -= 0.1;
else if (key == KEY_T)
cam->pitch -= 0.1;
else if (key == KEY_G)
cam->pitch += 0.1;
else if (key == KEY_R)
cam->roll += 0.1;
else if (key == KEY_Y)
cam->roll -= 0.1;

make_rotation_z(orientation, cam->roll);
make_rotation_x(rotation, cam->pitch);
mult_matrix(orientation, rotation, orientation);
make_rotation_y(rotation, cam->yaw);
mult_matrix(orientation, rotation, orientation);

s->current_cam->right = orientation[0];
s->current_cam->up = orientation[1];
s->current_cam->forward = orientation[2];
normalize(&s->current_cam->right);
normalize(&s->current_cam->up);
normalize(&s->current_cam->forward);
}


I increment/decrement the angles by pressing keys on the keyboard and then I reconstruct the matrix.

The problem is that this method assumes that the camera has direction (0, 0, -1), right (1, 0, 0, 0) and up (0, 1, 0).

When another camera is not initialized like this one but with a random orientation, as soon as I press a key it rotates from the camera directed to (0, 0, -1). (Because all three angles are initialized to 0).  I can't figure out how to initialize my angles with my camera.

I tried to calculate the angles this way:

float my_atan(float y, float x)
{
float res;

res = 0;
if (x > 0)
res = atan(y / x);
else if (y >= 0 && x < 0)
res = M_PI + atan(y / x);
else if (y < 0 && x < 0)
res = -M_PI + atan(y / x);
else if (y > 0 && x == 0)
res = M_PI / 2;
else if (y < 0 && x == 0)
res = -M_PI / 2;
else if (y == 0 && x == 0)
res = 0;
else if (y == 0 && x < 0)
res = res = 2 * M_PI;
if (res < 0)
res += 2 * M_PI;
return (res);
}

float phi = my_atan(cam->vec.y, cam->vec.x);
float theta = atan(sqrt(cam->vec.x * cam->vec.x + cam->vec.y * cam->vec.y) / cam->vec.z);

cam->yaw = theta;
cam->pitch = phi;
cam->roll = 0;


[EDIT] I use this function to initialise my angles :

void    init_camera_angles(t_camera *cam)
{
cam->yaw = atan2(cam->vec.x, -cam->vec.z);
cam->pitch = asin(cam->vec.y / -sqrt(get_norm_2(cam->vec)));
cam->roll = 0;
}


And I use this function to wrap the angles :

float           wrap_angle(float angle)
{
float two_pi;

two_pi = 2 * M_PI;
return (angle - two_pi * floor((angle + M_PI) / two_pi));
}


I have a problem at the beginning of the programme when I initialise the three vectors of my camera. I use this function to initialise these three vectors:

void    init_camera_axis(t_camera *cam)
{
t_vec3 up_tmp;

cam->forward = get_normalised(invert(cam->vec));
if (fabs(cam->vec.y) > 0.7)
{
if (cam->vec.z <= 0)
up_tmp = (t_vec3){0, 0, 1};
else
up_tmp = (t_vec3){0, 0, -1};
}
else
up_tmp = (t_vec3){0, 1, 0};
cam->right = vec_cross(up_tmp, cam->forward);
cam->up = vec_cross(cam->forward, cam->right);
}


The problem is that I get a right equal to (0.85, 0, 0) instead of (1, 0, 0), which produces the effect of a different zoom on the first image compared to the others generated with move_cam().

Maybe I'm doing the initialization of the three vectors wrong or I need to use another method?

• Can you describe what problems you're having with your current code? It's hard to tell from the short video clip, but what we see there looks like what we'd expect from the rotation model you've chosen, where the camera is allowed to roll, and always rotates about its own local axes. See I'm rotating an object on two axes, so why does it keep twisting about the third axis? Aug 20, 2020 at 11:18
• – zoom
Aug 20, 2020 at 13:06
• Looks like you forgot to normalize your right vector after computing it from the cross product. Because your two inputs are not guaranteed to be perpendicular, the result is not guaranteed to be unit length, even though the inputs were unit length. Aug 26, 2020 at 18:20
• Oh yeah, that's an oversight on my part. Thanks a lot!!
– zoom
Aug 26, 2020 at 19:52
• It looks like your latest edit is a new question, that has nothing whatsoever to do with your original problem of yawing diagonally. Please post a new question, rather than hiding it at the bottom of an existing, answered question. Aug 27, 2020 at 14:15

You can store three float variables, pitch, yaw, and roll (or equivalently, package them together into a 3-vector), representing the orientation of your camera as Tait-Bryan angles (often called Euler angles in gamedev, though their formal mathematical definition is different).

When you press a key, increment/decrement the corresponding angle, and wrap it into the range $$\[ -\pi, \pi]\$$ to keep it well-behaved. (You may want to clamp pitch or roll into a narrower range to avoid gimbal lock when looking straight up, or confusing controls if you roll upside-down)

Then after any angle has changed, rebuild your camera matrix from scratch:

t_vec3  rotation[3];

// Roll about Z is "most local", so we'll apply it first.
make_rotation_z(orientation, roll);

// Pitch about X is "in-between", so we'll apply it second.
make_rotation_x(rotation, pitch);
mult_matrix(orientation, rotation, orientation);

// Yaw about Y is "most global", so we'll apply it last.
make_rotation_y(rotation, yaw);
mult_matrix(orientation, rotation, orientation);


Note that we completely stomp the old value of the camera orientation each time we update the angles. So the angles are the source of truth here, and the matrix is just an "unpacking" of those angles into a usable form.

Your look-at method, or other ways to initialize the camera orientation, should then set these angles (say, transforming a look-at vector into spherical coordinates, and using those angles as your pitch and yaw), to ensure the source of truth is updated accordingly.

Here's an example of how to do that:

void SetCameraAnglesToFace(t_vec3 direction) {

yaw = atan2(-direction.x, -direction.z)
pitch = asin(diretion.y/magnitude(direction));

// Not enough information in a single direction vector
// to specify a twist around the roll axis.
roll = 0;
}

• It works well. However, I need to invert my orientation[2] which corresponds to the camera vector otherwise I get a camera looking behind. I don’t need to invert right and up. Also, I have stored the angles in my structure rather than defining them in static because I have several cameras. For a camera that is not located at the origin and that doesn’t look in -z but in a random direction, as soon as I rotate it, it looks in line with the -z axis. This must be because I set my angles to 0. How do I compute/initialize the angles at the beginning according to the vector of my camera?
– zoom
Aug 21, 2020 at 9:27
• See the very last paragraph. Aug 21, 2020 at 11:13
• I've seen your last paragraph and I don't see how I can calculate the angles. Can you tell me more about the transformation to spherical coordinates ?
– zoom
Aug 21, 2020 at 12:45
• I've edited the answer to show a method to take a direction vector and convert it to yaw and pitch, assuming (pitch, yaw) = (0, 0) means looking in the direction (0, 0, -1), that positive yaw should act leftward/counter-clockwise about the upward/y+ axis, and positive pitch should rotate upward. If your coordinate conventions are different you might have to swap some signs. Aug 25, 2020 at 13:14
• This is called gimbal lock, and I already explained it in the answer I linked you to earlier. This is the last time I will answer a comment here. If you have a new question, ask a new question in a new post. Aug 28, 2020 at 11:16

Note that camfwd is solely defined by (yaw,pitch) so calculate that first.

Given fwd / left / up of the null-rotation (right-handed system)...

// yaw the forward vector.
vec3 projectedcamfwd = fwd * cosf(yaw) + left * sinf(yaw);
// pitch the forward vector.
vec3 camfwd = projectedcamfwd * cosf(pitch) + up * sinf(pitch);


If you never want camera roll, then camleft is always horizontal, so:

vec3 camleft = cross( up, camfwd ).normalized();
vec3 camup = cross( camfwd, camleft );


With that, you have all axes (rows) of the camera matrix. Just group them along with a translation vector, and you are done!

If you do want camera roll, we need more work:

camup = camup * cosf( roll ) + camleft * sinf( roll );
camleft = cross( camup, camfwd );


Let me stress, that this will compute your camera transformation matrix. If you were to render with that in e.g. OpenGL, you still need to get the view matrix from the camera matrix, but that is easy, as viewmat is simply the inverse of the cammat.

I recommend that you limit pitch in the range [ -0.4999 π, 0.4999 π ] as it will cause issues when looking straight up.