I am trying to make a system where I can outline sprites, but I can only get it to work correctly when I use a regular-shaped texture. Here is what an outlined regular texture looks like:
However, here is an irregular texture with the same outline code applied to it:
I must say that the non-working outline does use modified texture coordinates, but this shouldn't matter, as all pixels with alpha = 0 are discarded (the second image is transparent around the person), which should prevent the stencil buffer from receiving them. However, the stencil buffer still counts the discarded pixels as "written". Here are my shaders, all of which we the same across the two textures:
Texture Vertex:
#version 330 core
layout (location = 0) in vec2 aPos;
layout (location = 1) in vec2 aTexCoord;
out vec2 TexCoord;
uniform mat4 transform;
uniform mat4 view;
uniform mat4 projection;
void main()
{
gl_Position = projection * view * transform * vec4(aPos, 0.0, 1.0);
TexCoord = aTexCoord;
}
Texture Fragment:
#version 330 core
out vec4 FragColor;
in vec2 TexCoord;
uniform sampler2D ourTexture;
uniform vec4 color;
void main()
{
FragColor = texture(ourTexture, TexCoord) * color;
if (FragColor.a == 0) discard;
}
Outline Fragment (paired with texture vertex):
#version 330 core
out vec4 fragColor;
uniform vec4 color;
void main() {
fragColor = color;
if (fragColor.a == 0) discard;
}
Finally, here is the shared render code:
void c2m::client::gl::Sprite::render() {
// Bind framebuffer
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
// Resize the image buffer for different outline sizing
glBindTexture(GL_TEXTURE_2D, texBuffer);
sf::Vector2u size = tex->getSfTexture().lock()->getSize();
GLuint sizeX = size.x + (GLuint)(size.x * outlineWidth * 2), sizeY = size.y + (GLuint)(size.y * outlineWidth * 2);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, sizeX, sizeY, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
// Resize renderbuffer?
// Clear framebuffer
glClearColor(.1f, .1f, .1f, 1.f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
// Set shader uniforms if the shader is initialized
if (shader != nullptr) {
if (outlineWidth > 0) {
glStencilFunc(GL_ALWAYS, 1, 0xFF);
glStencilMask(0xFF);
}
applyTransforms();
shader->useShader();
// Framebuffer will be transformed, this will not
shader->setMat4("transform", glm::mat4(1.0f));
shader->setVec4("color", color.asVec4());
}
// Rebind the VAO to be able to modify its VBOs
glBindVertexArray(vao);
// Reset vertex data to class array
glBindBuffer(GL_ARRAY_BUFFER, vbo);
glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices);
glBindBuffer(GL_ARRAY_BUFFER, 0);
// Bind texture
tex->bind();
// Draw
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
// If the outline width is 0 return
if (outlineWidth == 0) {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
return;
}
// Draw outline
if (outlineShader != nullptr) {
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilMask(0x00);
glDisable(GL_DEPTH_TEST);
outlineShader->useShader();
outlineShader->setVec4("color", outlineRGBA.asVec4());
// Temporary transform matrix to prevent pollution of user-set transforms
glm::mat4 tempTransform = trans;
tempTransform = glm::scale(tempTransform, glm::vec3(outlineWidth + 1, outlineWidth + 1, outlineWidth + 1));
tempTransform = glm::translate(tempTransform, glm::vec3(0, 0, 1));
// Framebuffer will be transformed, this will not
outlineShader->setMat4("transform", glm::mat4(1.0f));
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
glStencilMask(0xFF);
glEnable(GL_DEPTH_TEST);
glStencilFunc(GL_ALWAYS, 1, 0xFF);
}
glBindFramebuffer(GL_FRAMEBUFFER, 0);
frame->useShader();
frame->setMat4("transform", trans);
glBindTexture(GL_TEXTURE_2D, texBuffer);
// Temporarily use old VBO, will make a new one once textures are working
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
glBindTexture(GL_TEXTURE_2D, 0);
}
Here is my stencil buffer initialization code:
// Stencil
glEnable(GL_STENCIL_TEST);
// Disable stencil writing by default, to be enabled per draw cycle
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
Am I doing something wrong in my texture fragment shader that is not properly discarding transparent pixels, have I not configured the stencil buffer correctly, or did I miss something somewhere else?
UPDATE 1
Here is my renderbuffer init code:
glGenFramebuffers(1, &framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
glGenTextures(1, &texBuffer);
glBindTexture(GL_TEXTURE_2D, texBuffer);
sf::Vector2u size = tex->getSfTexture().lock()->getSize();
GLuint sizeX = size.x + (GLuint)(size.x * outlineWidth * 2), sizeY = size.y + (GLuint)(size.y * outlineWidth * 2);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, sizeX, sizeY, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texBuffer, 0);
glGenRenderbuffers(1, &rbo);
glBindRenderbuffer(GL_RENDERBUFFER, rbo);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, sizeX, sizeY);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, rbo);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
throw new std::exception("ERROR::RENDER::INCOMPLETE_FRAMEBUFFER");
return;
}
glBindFramebuffer(GL_FRAMEBUFFER, 0);
I have updated my render()
method and my vertex shader to represent my current code.
glStencilOp
may be problematic, but I'm not sure if it's the problem or how I could fix it if it was. \$\endgroup\$ – ImTheSquid Apr 7 '20 at 0:50glStencilOp
, as it could also beglStencilFunc
causing the issue. Overall, this is incredibly confusing, and I really hope OpenGL isn't doing something dumb in the background. \$\endgroup\$ – ImTheSquid Apr 7 '20 at 1:00