OpenGL blending (masking)

Question

I need some help with OpenGL textures masking. I have it working but need to find some other blending function parameters to work in other way. Now I have:

//Background 
...code...
    glBlendFunc(GL_ONE, GL_ZERO);
...code

//Mask
...code...
    glBlendFuncSeparate(GL_ZERO, GL_ONE, GL_DST_COLOR, GL_ZERO);
...code...

//Foreground
...code
    glBlendFunc(GL_DST_ALPHA, GL_ONE_MINUS_DST_ALPHA);
...code

Now it sets foreground's opacity to 0 (fills with background texture) where mask is transparent. I need it to react to mask's colors. I mean something like setting foregrounds opacity depending on mask's color. For example if mask is black (0.0,0.0,0.0) then the opacity of that place in foreground is 0 (is filled with background), and if mask is white (1.0,1.0,1.0) then the opacity of foreground is 1 (not filled with background). It can be in reverse consequence (white = opacity 0, black = opacity 1). I just need it to work depending on color.

---

1st column is my current result. Circle in mask is transparent.
2nd column is example of result I am trying to get. Circle in mask is white.
3rd column is example of why I want to get it working just like as I said (white mask color = foreground alpha 0, black mask color = foreground alpha 1 or reverse)

1st row is background
2nd row is mask
3rd row is foreground
4th row is result

Answer 1

Anyway you'll have to use shaders at sometime, because fixed-pipeline is deprecated now.

Lighthouse3D has good GLSL tutorials to start with. There is also tutorial for similar problem to yours.

Shader for your problem:

uniform sampler2D backTex,maskTex,foreText;

void main()
{
   float mask;
   vec4 result;

   mask = texture2D(maskTex,gl_TexCoord[0].st).r;
   result = texture2D(backTex,gl_TexCoord[0].st) * (1 - mask);
   result = result + texture2D(foreTex,gl_TexCoord[0].st) * mask;

   result.a = 1.0;

   gl_FragColor = result;
}

Edit: I've written whole masking application, 100% working.

#include <iostream>

#include <GL/glew.h>
#include <GL/glfw.h>
#include <IL/il.h>

using namespace std;

GLuint loadImage ( const char *filename );
extern const char *fsrc, *vsrc;

int main ( int argc, char **argv ) {
    glfwInit();
    glfwOpenWindow ( 800,600,8,8,8,8,8,0,GLFW_WINDOW );

    glewInit();
    ilInit();

    // CREATE TEXTURES
    GLuint background,mask,foreground;

    background = loadImage ( "background.png" );
    mask = loadImage ( "mask.png" );
    foreground = loadImage ( "foreground.png" );

    // CREATE SHADERS
    GLuint fragment,vertex,program;

    fragment = glCreateShader ( GL_FRAGMENT_SHADER );
    vertex = glCreateShader ( GL_VERTEX_SHADER );
    program = glCreateProgram();

    glShaderSource ( fragment, 1, &fsrc, 0 );
    glShaderSource ( vertex, 1, &vsrc, 0 );

    glCompileShader ( fragment );
    glCompileShader ( vertex );

    glAttachShader ( program, fragment );
    glAttachShader ( program, vertex );

    glLinkProgram ( program );

    glUseProgram ( program );

    // BIND TEXTURES TO TEXTURE UNITS
    glActiveTexture ( GL_TEXTURE0 );
    glEnable ( GL_TEXTURE_2D );
    glBindTexture ( GL_TEXTURE_2D,foreground );

    glActiveTexture ( GL_TEXTURE1 );
    glEnable ( GL_TEXTURE_2D );
    glBindTexture ( GL_TEXTURE_2D,mask );

    glActiveTexture ( GL_TEXTURE2 );
    glEnable ( GL_TEXTURE_2D );
    glBindTexture ( GL_TEXTURE_2D,background );

    // SET TEXTURE LOCATIONS FOR SHADER PROGRAM
    glUniform1i ( glGetUniformLocation ( program, "foregroundTex" ), 0 );
    glUniform1i ( glGetUniformLocation ( program, "maskTex" ), 1 );
    glUniform1i ( glGetUniformLocation ( program, "backgroundTex" ), 2 );

    glMatrixMode ( GL_PROJECTION );
    glLoadIdentity();
    glOrtho ( 0,1,0,1,-1,1 );
    glMatrixMode ( GL_MODELVIEW );
    glLoadIdentity();

    while ( glfwGetWindowParam ( GLFW_OPENED ) ) {

        glClear ( GL_COLOR_BUFFER_BIT );

        glBegin ( GL_QUADS );
        glColor4f ( 1,1,1,1 );
        glTexCoord2f ( 0,0 );
        glVertex2f ( 0,0 );
        glTexCoord2f ( 1,0 );
        glVertex2f ( 1,0 );
        glTexCoord2f ( 1,1 );
        glVertex2f ( 1,1 );
        glTexCoord2f ( 0,1 );
        glVertex2f ( 0,1 );
        glEnd();

        glfwSwapBuffers();
    }

    return 0;
}

GLuint loadImage ( const char* filename ) {
    GLuint tex = 0;
    ILuint texid;
    ilGenImages ( 1, &texid );
    ilBindImage ( texid );
    bool success = ilLoadImage ( ( const ILstring ) filename );
    if ( success ) { /* If no error occured: */
        success = ilConvertImage ( IL_RGB, IL_UNSIGNED_BYTE );
        if ( !success ) {
            cerr << "Failed to convert texture\n";
            return 0;
        }

        glGenTextures ( 1, &tex );
        glBindTexture ( GL_TEXTURE_2D, tex );
        glTexParameteri ( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
        glTexParameteri ( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
        glTexImage2D ( GL_TEXTURE_2D, 0, ilGetInteger ( IL_IMAGE_BPP ), ilGetInteger ( IL_IMAGE_WIDTH ),
                       ilGetInteger ( IL_IMAGE_HEIGHT ), 0, ilGetInteger ( IL_IMAGE_FORMAT ), GL_UNSIGNED_BYTE,
                       ilGetData() );
    } else {
        cerr << "Failed to load texture\n";
        return 0;
    }
    ilDeleteImages ( 1, &texid );

    return tex;
}

const char *fsrc = "\
uniform sampler2D foregroundTex, maskTex, backgroundTex; \
\
void main() {\
   float mask;\
   vec4 result;\
   mask = texture2D(maskTex,gl_TexCoord[0].st).r;\
   result = texture2D(backgroundTex,gl_TexCoord[0].st) * (1.0f - mask);\
   result = result + texture2D(foregroundTex,gl_TexCoord[0].st) * mask;\
   result.a = 1.0;\
   gl_FragColor = vec4(result);\
}";

const char *vsrc = "\
void main() { \
gl_Position = gl_ProjectionMatrix * gl_ModelViewMatrix * gl_Vertex;\
gl_TexCoord[0]  = gl_MultiTexCoord0;\
}";