Rotating camera in 3D without yawing diagonally

Question

I'm programming a C raytracer. I'm having trouble getting the camera to rotate the way I intend.

I would like the camera to rotate by pressing keys:

• F & H rotate left & right around the world's vertical axis (Y)

• T & G rotate up & down around an axis that is always horizontal and perpendicular to the view axis.

• R & Y rotate around the camera's viewing axis (local Z)

But with my current code, rotating up and then left rotates on a diagonal, instead of going around the world Y axis:

How can I make this rotation control more intuitive?

---

This is how the camera axes are initialized at the beginning of the program (I do this once) :

void    init_camera_vecs(t_camera *cam)
{
    t_vec3  forward;

    forward = get_normalised(invert(cam->vec));
    cam->up = (t_vec3){0, 0, 0};
    if (fabs(cam->vec.y) > 0.7)
        cam->right = vec_cross((t_vec3){0, 0, -1}, forward);
    else
        cam->right = vec_cross((t_vec3){0, 1, 0}, forward);
    cam->up = vec_cross(forward, cam->right);
}

This is my get_ray_dir() function which gives the direction of the ray from camera space to world space:

static t_vec3   lookat(t_camera *cam, t_vec3 ray_coord)
{
    t_vec3  forward;
    t_vec3  ray_dir;

    forward = get_normalised(invert(cam->vec));
    ray_dir.x = ray_coord.x * cam->right.x + ray_coord.y * cam->up.x
            + ray_coord.z * forward.x;
    ray_dir.y = ray_coord.x * cam->right.y + ray_coord.y * cam->up.y
            + ray_coord.z * forward.y;
    ray_dir.z = ray_coord.x * cam->right.z + ray_coord.y * cam->up.z
            + ray_coord.z * forward.z;
    return (ray_dir);
}

static t_vec3   get_ray_dir(int i, int j, t_resolution res, t_camera *cam)
{
    t_ray ray;
    float scale;
    float point_x;
    float point_y;
    float ratio;

    ratio = res.x / (float)res.y;
    scale = tan(cam->fov / 2 * M_PI / 180);
    point_x = (2 * (j + 0.5) / (float)res.x - 1) * ratio * scale;
    point_y = (1 - 2 * (i + 0.5) / res.y) * scale;
    ray.coord = (t_vec3){point_x, point_y, -1};
    ray.dir = lookat(cam, ray.coord);
    normalize(&ray.dir);
    return (ray.dir);
}

I tried this :

void    make_rotation_x(t_vec3 *rot_x, float angle)
{
    float   c;
    float   s;

    c = cos(angle);
    s = sin(angle);
    rot_x[0] = (t_vec3){1, 0, 0};
    rot_x[1] = (t_vec3){0, c, -s};
    rot_x[2] = (t_vec3){0, s, c};
}

void    make_rotation_y(t_vec3 *rot_y, float angle)
{
    float   c;
    float   s;

    c = cos(angle);
    s = sin(angle);
    rot_y[0] = (t_vec3){c, 0, s};
    rot_y[1] = (t_vec3){0, 1, 0};
    rot_y[2] = (t_vec3){-s, 0, c};
}

void    make_rotation_z(t_vec3 *rot_z, float angle)
{
    float   c;
    float   s;

    c = cos(angle);
    s = sin(angle);
    rot_z[0] = (t_vec3){c, -s, 0};
    rot_z[1] = (t_vec3){s, c, 0};
    rot_z[2] = (t_vec3){0, 0, 1};
}

void    mult_matrix(t_vec3 *res, t_vec3 *lhs, t_vec3 *rhs)
{
    res[0] = rotate_vector(lhs, rhs[0]);
    res[1] = rotate_vector(lhs, rhs[1]);
    res[2] = rotate_vector(lhs, rhs[2]);
}

void    cpy_matrix(t_vec3 *dst, t_vec3 *src)
{
    dst[0] = src[0];
    dst[1] = src[1];
    dst[2] = src[2];
}

void    apply_rotation_x(t_vec3 *orientation, float angle)
{
    t_vec3  matrix[3];
    t_vec3  res[3];

    make_rotation_x(matrix, angle);
    mult_matrix(res, orientation, matrix);
    cpy_matrix(orientation, res);
}

void    apply_rotation_y(t_vec3 *orientation, float angle)
{
    t_vec3  matrix[3];
    t_vec3  res[3];

    make_rotation_y(matrix, angle);
    mult_matrix(res, orientation, matrix);
    cpy_matrix(orientation, res);
}

void    apply_rotation_z(t_vec3 *orientation, float angle)
{
    t_vec3  matrix[3];
    t_vec3  res[3];

    make_rotation_z(matrix, angle);
    mult_matrix(res, orientation, matrix);
    cpy_matrix(orientation, res);
}

void        move_cam(t_specs *s, int key)
{
    t_camera    *cam;
    t_vec3      orientation[3];
    float       angle;

    angle = 4.5 * (M_PI / 180);
    cam = s->current_cam;
    orientation[0] = cam->right;
    orientation[1] = cam->up;
    orientation[2] = cam->vec;
    if (key == KEY_W)
        cam->coord = vec_add(cam->coord, cam->up);
    else if (key == KEY_S)
        cam->coord = vec_sub(cam->coord, cam->up);
    else if (key == KEY_A)
        cam->coord = vec_sub(cam->coord, cam->right);
    else if (key == KEY_D)
        cam->coord = vec_add(cam->coord, cam->right);
    else if (key == KEY_Q)
        cam->coord = vec_sub(cam->coord, cam->vec);
    else if (key == KEY_E)
        cam->coord = vec_add(cam->coord, cam->vec);
    else if (key == KEY_H)
        apply_rotation_y(orientation, -angle);
    else if (key == KEY_F)
        apply_rotation_y(orientation, angle);
    else if (key == KEY_G)
        apply_rotation_x(orientation, -angle);
    else if (key == KEY_T)
        apply_rotation_x(orientation, angle);
    else if (key == KEY_R)
        apply_rotation_z(orientation, -angle);
    else if (key == KEY_Y)
        apply_rotation_z(orientation, angle);
    s->current_cam->right = orientation[0];
    s->current_cam->up = orientation[1];
    s->current_cam->vec = orientation[2];
    normalize(&s->current_cam->right);
    normalize(&s->current_cam->up);
    normalize(&s->current_cam->vec);
}

I use this function now for rotation.

void        move_cam(t_specs *s, int key)
{
t_camera    *cam;
t_vec3      orientation[3];

cam = s->current_cam;
orientation[0] = cam->right;
orientation[1] = cam->up;
orientation[2] = cam->forward;

if (key == KEY_W)
    cam->coord = vec_add(cam->coord, cam->up);
else if (key == KEY_S)
    cam->coord = vec_sub(cam->coord, cam->up);
else if (key == KEY_A)
    cam->coord = vec_sub(cam->coord, cam->right);
else if (key == KEY_D)
    cam->coord = vec_add(cam->coord, cam->right);
else if (key == KEY_Q)
    cam->coord = vec_sub(cam->coord, cam->vec);
else if (key == KEY_E)
    cam->coord = vec_add(cam->coord, cam->vec);
else if (key == KEY_H)
    cam->yaw += 0.1;
else if (key == KEY_F)
    cam->yaw -= 0.1;
else if (key == KEY_T)
    cam->pitch -= 0.1;
else if (key == KEY_G)
    cam->pitch += 0.1;
else if (key == KEY_R)
    cam->roll += 0.1;
else if (key == KEY_Y)
    cam->roll -= 0.1;

make_rotation_z(orientation, cam->roll);
make_rotation_x(rotation, cam->pitch);
mult_matrix(orientation, rotation, orientation);
make_rotation_y(rotation, cam->yaw);
mult_matrix(orientation, rotation, orientation);

s->current_cam->right = orientation[0];
s->current_cam->up = orientation[1];
s->current_cam->forward = orientation[2];
normalize(&s->current_cam->right);
normalize(&s->current_cam->up);
normalize(&s->current_cam->forward);
}

I increment/decrement the angles by pressing keys on the keyboard and then I reconstruct the matrix.

The problem is that this method assumes that the camera has direction (0, 0, -1), right (1, 0, 0, 0) and up (0, 1, 0).

When another camera is not initialized like this one but with a random orientation, as soon as I press a key it rotates from the camera directed to (0, 0, -1). (Because all three angles are initialized to 0).

I can't figure out how to initialize my angles with my camera.

I tried to calculate the angles this way:

float my_atan(float y, float x)
{
    float res;
    
    res = 0;
    if (x > 0)
        res = atan(y / x);
    else if (y >= 0 && x < 0)
        res = M_PI + atan(y / x);
    else if (y < 0 && x < 0)
        res = -M_PI + atan(y / x);
    else if (y > 0 && x == 0)
        res = M_PI / 2;
    else if (y < 0 && x == 0)
        res = -M_PI / 2;
    else if (y == 0 && x == 0)
        res = 0;
    else if (y == 0 && x < 0)
        res = res = 2 * M_PI;
    if (res < 0)
        res += 2 * M_PI;
    return (res);
}

float phi = my_atan(cam->vec.y, cam->vec.x);
float theta = atan(sqrt(cam->vec.x * cam->vec.x + cam->vec.y * cam->vec.y) / cam->vec.z);
    
cam->yaw = theta;
cam->pitch = phi;
cam->roll = 0;

[EDIT] I use this function to initialise my angles :

void    init_camera_angles(t_camera *cam)
{
    cam->yaw = atan2(cam->vec.x, -cam->vec.z);
    cam->pitch = asin(cam->vec.y / -sqrt(get_norm_2(cam->vec)));
    cam->roll = 0;
}

And I use this function to wrap the angles :

float           wrap_angle(float angle) 
{
    float two_pi;

    two_pi = 2 * M_PI;
    return (angle - two_pi * floor((angle + M_PI) / two_pi));
}

I have a problem at the beginning of the programme when I initialise the three vectors of my camera. I use this function to initialise these three vectors:

void    init_camera_axis(t_camera *cam)
{
    t_vec3 up_tmp;

    cam->forward = get_normalised(invert(cam->vec));
    if (fabs(cam->vec.y) > 0.7)
    {
        if (cam->vec.z <= 0)
        up_tmp = (t_vec3){0, 0, 1};
        else
            up_tmp = (t_vec3){0, 0, -1};
    }
    else
        up_tmp = (t_vec3){0, 1, 0};
    cam->right = vec_cross(up_tmp, cam->forward);
    cam->up = vec_cross(cam->forward, cam->right);
}

The problem is that I get a right equal to (0.85, 0, 0) instead of (1, 0, 0), which produces the effect of a different zoom on the first image compared to the others generated with move_cam().

Maybe I'm doing the initialization of the three vectors wrong or I need to use another method?

Answer 1

You can store three float variables, pitch, yaw, and roll (or equivalently, package them together into a 3-vector), representing the orientation of your camera as Tait-Bryan angles (often called Euler angles in gamedev, though their formal mathematical definition is different).

When you press a key, increment/decrement the corresponding angle, and wrap it into the range \$[ -\pi, \pi]\$ to keep it well-behaved. (You may want to clamp pitch or roll into a narrower range to avoid gimbal lock when looking straight up, or confusing controls if you roll upside-down)

Then after any angle has changed, rebuild your camera matrix from scratch:

t_vec3  rotation[3];

// Roll about Z is "most local", so we'll apply it first.
make_rotation_z(orientation, roll);

// Pitch about X is "in-between", so we'll apply it second.
make_rotation_x(rotation, pitch);
mult_matrix(orientation, rotation, orientation);

// Yaw about Y is "most global", so we'll apply it last.
make_rotation_y(rotation, yaw);
mult_matrix(orientation, rotation, orientation);

Note that we completely stomp the old value of the camera orientation each time we update the angles. So the angles are the source of truth here, and the matrix is just an "unpacking" of those angles into a usable form.

Your look-at method, or other ways to initialize the camera orientation, should then set these angles (say, transforming a look-at vector into spherical coordinates, and using those angles as your pitch and yaw), to ensure the source of truth is updated accordingly.

Here's an example of how to do that:

void SetCameraAnglesToFace(t_vec3 direction) {    
    
    yaw = atan2(-direction.x, -direction.z)
    pitch = asin(diretion.y/magnitude(direction));

    // Not enough information in a single direction vector
    // to specify a twist around the roll axis.
    roll = 0; 
}

Answer 2

Note that camfwd is solely defined by (yaw,pitch) so calculate that first.

Given fwd / left / up of the null-rotation (right-handed system)...

// yaw the forward vector.
vec3 projectedcamfwd = fwd * cosf(yaw) + left * sinf(yaw);
// pitch the forward vector.
vec3 camfwd = projectedcamfwd * cosf(pitch) + up * sinf(pitch);

If you never want camera roll, then camleft is always horizontal, so:

vec3 camleft = cross( up, camfwd ).normalized();
vec3 camup = cross( camfwd, camleft );

With that, you have all axes (rows) of the camera matrix. Just group them along with a translation vector, and you are done!

If you do want camera roll, we need more work:

camup = camup * cosf( roll ) + camleft * sinf( roll );
camleft = cross( camup, camfwd );

Let me stress, that this will compute your camera transformation matrix. If you were to render with that in e.g. OpenGL, you still need to get the view matrix from the camera matrix, but that is easy, as viewmat is simply the inverse of the cammat.

I recommend that you limit pitch in the range [ -0.4999 π, 0.4999 π ] as it will cause issues when looking straight up.