Cannot transform vectors from viewport to world

Question

I started working on a small raytracer project, and i decided to reuse my already existing openGL renderer to do this, i'm using GLM to manage transforms/positions.

However, i stumbled upon a very annoying issue: when transforming my rays from screen coordinates to world coordinates, i get very much wrong results. For instance, if i try to transform a vector v(0, 0, 0, 1) from screen to world by doing inverse(viewport * projection * view) * v, i am expecting to retrieve a point at the top left of my camera's near plane, but instead i am getting a point exetremely far away from it.

I don't understand what is happening, since the renderer works very well. I tracked the problem to be either the view() or projection() functions of my camera, but everything looks perfectly fine, i am missing a very important point that i am not aware of.

For reference, the camera class:

#pragma once
#include <glm/glm.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <sprout_engine/shader.h>
#include "inspectable.h"

using namespace glm;

enum CAMERA_DIR
{
    FORWARD,
    BACKWARD,
    LEFT,
    RIGHT,
    UP,
    DOWN
};

struct Plane
{
    glm::vec3 n;
    float d; // distance from origin to nearest point of the plane

    Plane() = default;
    Plane(const glm::vec3& p, const glm::vec3& norm) : n(glm::normalize(norm)), d(glm::dot(n, p)) {}

    inline float getSignedDistanceToPlane(const glm::vec3& p) const { return glm::dot(n, p) - d; };
};

struct Frustum
{
    Plane topFace;
    Plane bottomFace;

    Plane rightFace;
    Plane leftFace;

    Plane farFace;
    Plane nearFace;
};

const float CAMERA_SPEED = 5.f;
const float CAMERA_SENSITIVITY = .1f;

class Camera : public Inspectable
{
protected:
    glm::vec3 pos;
    glm::vec3 dir;
    glm::vec3 up;
    glm::vec3 worldUp;
    glm::vec3 right;

    float m_zNear;
    float m_zFar;
    float m_fov;
    float m_aspectRatio;

    float pitch;
    float yaw;

    glm::mat4 m_view{};
    glm::mat4 m_projection{};

    Frustum m_frustum;

    void update_dir();

public:
    Camera();
    Camera(const glm::vec3 &pos, const glm::vec3 &up, float pitch, float yaw, float p_znear, float p_zfar, float p_fov, float p_aspectRatio);

    void drawInspector() override;

    [[nodiscard]] glm::mat4 view() const;
    [[nodiscard]] glm::mat4 projection() const;

    [[nodiscard]] inline glm::vec3 get_position() const { return pos; };
    [[nodiscard]] inline Frustum getFrustum() const { return m_frustum; };

    void setZNear(float mZNear);
    void setZFar(float mZFar);
    void setFov(float mFov);
    void setAspectRatio(float mAspectRatio);

    void updateView();
    void updateProjection();
    void updateFrustum();

    void process_input(CAMERA_DIR direction, float delta_time);

    void process_mouse_movement(float xoffset, float yoffset);
};

and its implementation:

//
// Created by Bellaedris on 28/04/2024.
//

#include "camera.h"
#include "imgui/imgui.h"

void Camera::update_dir() {
    vec3 new_dir;
    float yawRad = radians(yaw);
    float pitchRad = radians(pitch);
    new_dir.x = std::cos(yawRad) * std::cos(pitchRad);
    new_dir.y = std::sin(pitchRad);
    new_dir.z = std::sin(yawRad) * std::cos(pitchRad);

    dir = normalize(new_dir);
    right = normalize(cross(dir, worldUp));
    up = normalize(cross(right, dir));

    updateView();
}

Camera::Camera()
        : dir(vec3(0., 0., -1.)), pitch(0.), yaw(0.)
{
    pos = vec3(0., 0., 0.);
    worldUp = vec3(0., 1., 0.);

    right = normalize(cross(dir, worldUp));
}

Camera::Camera(const vec3 &pos, const vec3 &up, float pitch, float yaw, float p_znear, float p_zfar, float p_fov,
               float p_aspectRatio)
        : dir(vec3(0., 0., -1.)), pos(pos), worldUp(up), up(up), pitch(pitch), yaw(yaw), m_zNear(p_znear), m_zFar(p_zfar), m_fov(p_fov), m_aspectRatio(p_aspectRatio)
{
    right = normalize(cross(dir, up));
    update_dir();

    updateView();
    updateProjection();
    updateFrustum();
}

glm::mat4 Camera::view() const {
    return m_view;
}

glm::mat4 Camera::projection() const {
    return m_projection;
}

void Camera::setZNear(float mZNear) {
    m_zNear = mZNear;
    updateProjection();
}

void Camera::setZFar(float mZFar) {
    m_zFar = mZFar;
    updateProjection();
}

void Camera::setFov(float mFov) {
    m_fov = mFov;
    updateProjection();
}

void Camera::setAspectRatio(float mAspectRatio) {
    m_aspectRatio = mAspectRatio;
    updateProjection();
}

void Camera::updateView() {
    m_view = glm::lookAt(pos, pos + dir, up);
    updateFrustum();
}

void Camera::updateProjection() {
    m_projection = glm::perspective(glm::radians(m_fov), m_aspectRatio, m_zNear, m_zFar);
    updateFrustum();
}

void Camera::updateFrustum() {
    Frustum frustum;
    float halfVSide = std::tan(glm::radians(m_fov) * .5f) * m_zFar; // find the half height of the far plane with trigo
    float halfHSide = halfVSide * m_aspectRatio; // aspect = w / h
    vec3 farPlaneCenter = m_zFar * dir;

    frustum.farFace = { pos + farPlaneCenter, -dir };
    frustum.nearFace = { pos + m_zNear * dir, dir };

    frustum.rightFace = { pos , cross(farPlaneCenter - right * halfHSide, up) };
    frustum.leftFace = { pos , cross(up, farPlaneCenter + right * halfHSide) };

    frustum.topFace = { pos , cross(right, farPlaneCenter - up * halfVSide) };
    frustum.bottomFace = { pos , cross(farPlaneCenter + up * halfVSide, right) };

    m_frustum = frustum;
}

void Camera::process_input(CAMERA_DIR direction, float delta_time) {
    float velocity = CAMERA_SPEED * delta_time;
    switch (direction)
    {
        case FORWARD:
            pos += dir * velocity;
            break;
        case BACKWARD:
            pos -= dir * velocity;
            break;
        case LEFT:
            pos -= right * velocity;
            break;
        case RIGHT:
            pos += right * velocity;
            break;
        case UP:
            pos += up * velocity;
            break;
        case DOWN:
            pos -= up * velocity;
            break;
    }
    update_dir();
}

void Camera::process_mouse_movement(float xoffset, float yoffset) {
    xoffset = std::abs(xoffset) <= 1.f ? 0.f : xoffset;
    yoffset = std::abs(yoffset) <= 1.f ? 0.f : yoffset;

    xoffset *= CAMERA_SENSITIVITY;
    yoffset *= CAMERA_SENSITIVITY;

    yaw += xoffset;
    pitch += yoffset;

    // to avoid the lookAt matrix to flip
    if (pitch > 89.f)
        pitch = 89.f;
    if (pitch < -89.f)
        pitch = -89.f;

    update_dir();
}

void Camera::drawInspector() {
    if(ImGui::TreeNode("Camera"))
    {
        if(ImGui::InputFloat3("Position", glm::value_ptr(pos)))
        {
            update_dir();
        }
        if (ImGui::InputFloat("Pitch", &pitch))
        {
            update_dir();
        }
        if (ImGui::InputFloat("Yaw", &yaw))
        {
            update_dir();
        }
        if (ImGui::InputFloat("FoV", &m_fov))
        {
            updateProjection();
        }
        ImGui::TreePop();
    }
}

The function that gives me the viewport matrix (m_width and m_height are the dimensions of the window)

glm::mat4 SproutApp::viewport() const {
    float w = (float)m_width / 2.f;
    float h = (float)m_height / 2.f;

    return {
            w,  0.,  0.,  0,
            0,  h,  0,   0,
            0., 0., .5f, 0,
            w,  h,  .5f,   1
            };
}

And the piece of code that produces wrong transformation:

m_model = Model(resources_path + "models/cornell-box.obj");
        cam = new Camera({0, 2, 5}, {0, 1, 0}, 0, -90.f, 0.1f, 100.f, 70.f, (float)width() / (float)height());
        m_shader = Shader("texture.vs", "texture.fs");
        m_debugShader = Shader("default.vs", "default.fs");

        m_lines = std::make_unique<LineRenderer>(cam);

        glm::mat4 camToWorld = glm::inverse(cam->view());
        glm::mat4 screenToWorld = glm::inverse(viewport() * cam->projection() * cam->view());

        m_lines->addLine({screenToWorld * glm::vec4(0, 0, 0, 1)}, {screenToWorld * glm::vec4(0, 0, 1, 1)});
        m_lines->addLine({screenToWorld * glm::vec4(width(), 0, 0, 1)}, {screenToWorld * glm::vec4(width(), 0, 1, 1)});
        m_lines->addLine({screenToWorld * glm::vec4(0, height(), 0, 1)}, {screenToWorld * glm::vec4(0, height(), 1, 1)});
        m_lines->addLine({screenToWorld * glm::vec4(width(), height(), 0, 1)}, {screenToWorld * glm::vec4(width(), height(), 1, 1)});
```