I have a camera follow script and I am rotating the camera with Q and E key to right and left. The Camera is a child of a camHolder object. I came up with this code, which rotates around the player to the right and left, but it's shaking as it tries to keep rotating around the player.

I didn't come up with a solution to stop the RotateAround function, instead I tried setting rotateSpeed to 0 after I let go of the key, but that didn't work either.

public Transform player;
public float smoothSpeed = 0.125f;
public float rotateSpeed;
private Vector3 velocity = Vector3.zero;

[SerializeField]
private Vector3 offset;

[Space]
public Transform camHolder;

void LateUpdate () 
{
    CameraFollowPlayer();
}

void Update()
{
    RotateCamera();
}

void CameraFollowPlayer()
{
    camHolder.position = Vector3.SmoothDamp(transform.position, player.position + offset, 
                                             ref velocity, smoothSpeed * Time.deltaTime);
    transform.LookAt(player.position);
}

void RotateCamera()
{
    if(Input.GetKey (KeyCode.E))
    {
        rotateSpeed += 20f;
        if(rotateSpeed >= 20f)
        {
            rotateSpeed = 20f;
        transform.RotateAround(player.position, transform.up, rotateSpeed * Time.deltaTime);
        }
    }
    else
    {
        rotateSpeed = 0f;
    }
    
    if(Input.GetKey (KeyCode.Q))
    {
        rotateSpeed += 20f;
        if(rotateSpeed >= 20f)
        {
            rotateSpeed = 20f;
            transform.RotateAround(player.position, transform.up, -rotateSpeed * Time.deltaTime);
        }
    }
    else
    {
        rotateSpeed = 0f;
    }
}