For anyone in the future looking for a solution to this using Unity/C#, here is an implementation I made based on Steve's answer:
//takes a rotation and returns the rotation that is the closest with all axes pointing at 90 degree intervals to the identity quaternion
Quaternion snapToNearestRightAngle(Quaternion currentRotation)
{
Vector3 closestToForward = closestToAxis(currentRotation, Vector3.forward);
Vector3 closestToUp = closestToAxis(currentRotation, Vector3.up);
return Quaternion.LookRotation(closestToForward, closestToUp);
}
//finds the axis that is closest to the currentRotations local axis
Vector3 closestToAxis(Quaternion currentRotation, Vector3 axis)
{
Vector3[] checkAxes = new Vector3[] { Vector3.forward, Vector3.right, Vector3.up, -Vector3.forward, -Vector3.right, -Vector3.up };
Vector3 closestToAxis = Vector3.forward;
float highestDot = -1;
foreach (Vector3 checkAxis in checkAxes)
check(ref highestDot, ref closestToAxis, currentRotation, axis, checkAxis);
return closestToAxis;
}
//finds the closest axis to the input rotations specified axis
void check(ref float highestDot, ref Vector3 closest, Quaternion currentRotation, Vector3 axis, Vector3 checkDir)
{
float dot = Vector3.Dot(currentRotation * axis, checkDir);
if (dot > highestDot)
{
highestDot = dot;
closest = checkDir;
}
}
And here is an implementation based on this post which is much more efficient:
//takes a rotation and returns the rotation that is the closest with all axes pointing at 90 degree intervals to the identity quaternion
Quaternion snapToNearestRightAngle(Quaternion currentRotation)
{
Vector3 closestToForward = SnappedToNearestAxis(currentRotation * Vector3.forward);
Vector3 closestToUp = SnappedToNearestAxis(currentRotation * Vector3.up);
return Quaternion.LookRotation(closestToForward, closestToUp);
}
//find the world axis that is closest to direction
Vector3 SnappedToNearestAxis(Vector3 direction)
{
float x = Mathf.Abs(direction.x);
float y = Mathf.Abs(direction.y);
float z = Mathf.Abs(direction.z);
if (x > y && x > z)
{
return new Vector3(Mathf.Sign(direction.x), 0, 0);
}
else if (y > x && y > z)
{
return new Vector3(0, Mathf.Sign(direction.y), 0);
}
else
{
return new Vector3(0, 0, Mathf.Sign(direction.z));
}
}