Incorrect GPU skinning

Question

I'm having some problems with implementing skinning and skeletal animation. It seems that the skeleton and the mesh are loaded correctly, but the mesh gets funky when the bone transformations are applied. What could be the problem?

---

Images depicting the scenario

Here's my original model made in Blender:

And here's the bone structure:

This is what the skeleton looks like in the bind pose when I load it, each joint is depicted as a red sphere. I seems to be correct:

And this is with the mesh, without transforming the vertices. Again, seems normal (note that the camera is rotated -30 degrees around the X axis, so the mesh is lying on the ground and the skeleton is standing up):

And finally, here's the vertex transformation in action. This is where it gets ugly:

---

As you can see from the images, everything goes nicely until the bones are actually transforming the vertices. Here's some relevant code, I hope the variable names are self-explanatory, and as I try to rubber-duck everything, I try to include the comments about what I think I'm doing.

Vertex struct

// Vertex struct
struct Vertex
{
    public Vector3 position; // Position
    public Vector3 normal;   // Normal
    public Vector2 uv;       // UV coordinate
    public Vector3 tangent;  // Tangent
    public Vector3 bitangent; // Bitangent
    public int bone0, bone1, bone2, bone3; // Bone indices
    public Vector4 boneWeights; // Bone weights

    // Retrieves the size of the struct in bytes
    public int Size()
    {
        return 4 * Vector3.SizeInBytes + Vector2.SizeInBytes + Vector4.SizeInBytes + 4 * sizeof(int);
    }
}

---

Mesh creation

// ** Create the mesh **
// Vertices
id_vb = GL.GenBuffer();
GLStates.UseVertexBuffer(id_vb); // equivalent to glBindBuffer
GL.VertexAttribPointer(6, 4, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes * 4 + Vector2.SizeInBytes + sizeof(int) * 4);
GL.VertexAttribPointer(5, 4, VertexAttribPointerType.Int, true, Vertex.Size(), Vector3.SizeInBytes * 4 + Vector2.SizeInBytes);
GL.VertexAttribPointer(4, 3, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes * 3 + Vector2.SizeInBytes);
GL.VertexAttribPointer(3, 3, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes * 2 + Vector2.SizeInBytes);
GL.TexCoordPointer(2, TexCoordPointerType.Float, Vertex.Size(), Vector3.SizeInBytes * 2);
GL.NormalPointer(NormalPointerType.Float, Vertex.Size(), Vector3.SizeInBytes);
GL.VertexPointer(3, VertexPointerType.Float, Vertex.Size(), 0);

// Indices
id_ib = GL.GenBuffer();
GLStates.UseElementBuffer(id_ib);
GL.BufferData(BufferTarget.ElementArrayBuffer, (IntPtr)(sizeof(uint) * indices.Count), indices.ToArray(), BufferUsageHint.StaticDraw);

---

Uploading bone transformations to the vertex shader

// Upload bone transformations
for (int i = 0; i < e.Skeleton.bones.Count && i < 16; ++i)
{
    // InverseLink is the inverse bind pose
    Matrix4 boneTransform = e.Skeleton.bones[i].InverseLink * e.Skeleton.bones[i].Entity.worldTransform.Matrix;

    // Upload the bone transform matrix
    GL.UniformMatrix4(Shader.currentShader.GetUniformLocation("boneTransforms[" + i + "]"), false, ref boneTransform);
}

---

Render the mesh

GLStates.UseVertexBuffer(id_vb);
GLStates.UseElementBuffer(id_ib);

// Re-upload the vertex data
GL.BufferData(BufferTarget.ArrayBuffer, (IntPtr)(Vertex.Size() * vertices.Count), vertices.ToArray(), BufferUsageHint.StreamDraw);

// Enable vertex arrays
GL.EnableVertexAttribArray(6);
GL.EnableVertexAttribArray(5);
GL.EnableVertexAttribArray(4);
GL.EnableVertexAttribArray(3);
GL.EnableVertexAttribArray(2);
GL.EnableVertexAttribArray(1);
GL.EnableVertexAttribArray(0);

GL.VertexAttribPointer(6, 4, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes * 4 + Vector2.SizeInBytes + sizeof(int) * 4);
GL.VertexAttribPointer(5, 4, VertexAttribPointerType.Int, true, Vertex.Size(), Vector3.SizeInBytes * 4 + Vector2.SizeInBytes);
GL.VertexAttribPointer(4, 3, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes * 3 + Vector2.SizeInBytes);
GL.VertexAttribPointer(3, 3, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes * 2 + Vector2.SizeInBytes);
GL.VertexAttribPointer(2, 2, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes * 2);
GL.VertexAttribPointer(1, 3, VertexAttribPointerType.Float, true, Vertex.Size(), Vector3.SizeInBytes);
GL.VertexAttribPointer(0, 3, VertexAttribPointerType.Float, true, Vertex.Size(), 0);

// And finally, draw the mesh
GL.DrawElements(BeginMode.Triangles, indices.Count, DrawElementsType.UnsignedInt, 0);

---

Vertex shader

#version 330

uniform mat4 projection;
uniform mat4 view;
uniform mat4 world;

uniform vec3 diffuseColor;

uniform mat4 boneTransforms[16];

layout(location = 0) in vec3 in_position;
layout(location = 1) in vec3 in_normal;
layout(location = 2) in vec2 in_texcoord;
layout(location = 3) in vec3 in_tangent;
layout(location = 4) in vec3 in_binormal;
layout(location = 5) in vec4 in_boneIndex;
layout(location = 6) in vec4 in_boneWeight;

out mat3 tangentToWorld;
out vec2 texcoord;
out vec3 color;

void main(void)         
{   
    // Use this if you do NOT want to use bones
    // gl_Position = projection * view * world * vec4(in_position, 1.0);

    // Get the vertex position
    vec4 vPos = vec4(in_position, 1.0);

    // Declare the bone vectors
    vec4 b0 = vec4(0);
    vec4 b1 = vec4(0);
    vec4 b2 = vec4(0);
    vec4 b3 = vec4(0);

    // If bone 0 is to be used
    if(in_boneIndex.x != -1)
    {
        // Transform by bone 0 matrix, weight by bone 0 weight
        b0 = (boneTransforms[int(in_boneIndex.x)] * vPos) * in_boneWeight.x;
    }
    // If bone 1 is... and so on
    if(in_boneIndex.y != -1)
    {
        b1 = (boneTransforms[int(in_boneIndex.y)] * vPos) * in_boneWeight.y;
    }
    if(in_boneIndex.z != -1)
    {
        b2 = (boneTransforms[int(in_boneIndex.z)] * vPos) * in_boneWeight.z;
    }
    if(in_boneIndex.w != -1)
    {
        b3 = (boneTransforms[int(in_boneIndex.w)] * vPos) * in_boneWeight.w;
    }

    // Finally, apply the WVP transform to the newly-acquired vertex position
    gl_Position = projection * view * world * vec4(((b0 + b1 + b2 + b3).xyz), 1.0);

    // Pass the color
    color = diffuseColor;

    // Transform NBT to world
    tangentToWorld[0] = (world * vec4(in_tangent, 0.0)).xyz;
    tangentToWorld[1] = (world * vec4(in_binormal, 0.0)).xyz;
    tangentToWorld[2] = (world * vec4(in_normal, 0.0)).xyz;
}

---

I guess there's all the relevant code. I'm more than happy to supply you with more code and information in case I missed something.

Answer 1

From a quick once-over it looks like all your stuff is correct conceptually. However, your vertex shader looks a bit suspect though. It seems like you are applying the world transform twice since it is already contained in the bone transforms. Or at least it seems that way in your code. This would be valid though if you bone transforms were relative to your overall object transform.

Also, I was initially confused by the fact that you were transforming by each bone influence and then adding the positions versus building a combined matrix first. I'm too lazy to work it out, but I think they are identical mathematically so this could just be a personal style thing.

The way I would usually do it is this:

// Declare the bone vectors
mat4 worldSkinned = mat4(0);

// If bone 0 is to be used
if(in_boneIndex.x != -1)
{
    worldSkinned += boneTransforms[int(in_boneIndex.x)] * in_boneWeight.x;
}
if(in_boneIndex.y != -1)
{
    worldSkinned += boneTransforms[int(in_boneIndex.y)] * in_boneWeight.y;
}
if(in_boneIndex.z != -1)
{
    worldSkinned += boneTransforms[int(in_boneIndex.z)] * in_boneWeight.z;
}
if(in_boneIndex.w != -1)
{
    worldSkinned += boneTransforms[int(in_boneIndex.w)] * in_boneWeight.w;
}

gl_Position = projection * view * worldSkinned * vPos;

However, I prefer this way because you also need to transform the TBN basis and this way you can reuse the same matrix if you don't care about non-uniform scales

// Transform NBT to world
tangentToWorld[0] = (worldSkinned * vec4(in_tangent, 0.0)).xyz;
tangentToWorld[1] = (worldSkinned * vec4(in_binormal, 0.0)).xyz;
tangentToWorld[2] = (worldSkinned * vec4(in_normal, 0.0)).xyz;

If you do need to care about non-uniform scales then you need to

mat3 worldSkinnedNormal = transpose(inverse(mat3(worldSkinned)));
tangentToWorld[0] = (worldSkinnedNormal * in_tangent).xyz;
tangentToWorld[1] = (worldSkinnedNormal * in_binorma).xyz;
tangentToWorld[2] = (worldSkinnedNormal * in_normal).xyz;

Or just pass a second boneTransformsInverseTranspose[] to the shader, which could be faster depending on whether you are bottlenecked on uniforms or shader math.

EDIT: Also, another thing I noticed is that your index attribute seems weird. You have a float shader input but you are using glVertexAttribPointer with type GL_INT but with normalization turned on. So you are actually getting float(idx)/float(INT_MAX) in your shader. You should try passing false for the fourth argument or alternatively using an ivec4 in your shader and glVertexAttribIPointer directly.