Anyway you'll have to use shaders at sometime, because fixed-pipeline is deprecated now.
Lighthouse3D has good GLSL tutorials to start with. There is also tutorial for similar problem to yours.
Shader for your problem:
uniform sampler2D backTex,maskTex,foreText;
void main()
{
float mask;
vec4 result;
mask = texture2D(maskTex,gl_TexCoord[0].st).r;
result = texture2D(backTex,gl_TexCoord[0].st) * (1 - mask);
result = result + texture2D(foreTex,gl_TexCoord[0].st) * mask;
result.a = 1.0;
gl_FragColor = result;
}
Edit:
I've written whole masking application, 100% working.
#include <iostream>
#include <GL/glew.h>
#include <GL/glfw.h>
#include <IL/il.h>
using namespace std;
GLuint loadImage ( const char *filename );
extern const char *fsrc, *vsrc;
int main ( int argc, char **argv ) {
glfwInit();
glfwOpenWindow ( 800,600,8,8,8,8,8,0,GLFW_WINDOW );
glewInit();
ilInit();
// CREATE TEXTURES
GLuint background,mask,foreground;
background = loadImage ( "background.png" );
mask = loadImage ( "mask.png" );
foreground = loadImage ( "foreground.png" );
// CREATE SHADERS
GLuint fragment,vertex,program;
fragment = glCreateShader ( GL_FRAGMENT_SHADER );
vertex = glCreateShader ( GL_VERTEX_SHADER );
program = glCreateProgram();
glShaderSource ( fragment, 1, &fsrc, 0 );
glShaderSource ( vertex, 1, &vsrc, 0 );
glCompileShader ( fragment );
glCompileShader ( vertex );
glAttachShader ( program, fragment );
glAttachShader ( program, vertex );
glLinkProgram ( program );
glUseProgram ( program );
// BIND TEXTURES TO TEXTURE UNITS
glActiveTexture ( GL_TEXTURE0 );
glEnable ( GL_TEXTURE_2D );
glBindTexture ( GL_TEXTURE_2D,foreground );
glActiveTexture ( GL_TEXTURE1 );
glEnable ( GL_TEXTURE_2D );
glBindTexture ( GL_TEXTURE_2D,mask );
glActiveTexture ( GL_TEXTURE2 );
glEnable ( GL_TEXTURE_2D );
glBindTexture ( GL_TEXTURE_2D,background );
// SET TEXTURE LOCATIONS FOR SHADER PROGRAM
glUniform1i ( glGetUniformLocation ( program, "foregroundTex" ), 0 );
glUniform1i ( glGetUniformLocation ( program, "maskTex" ), 1 );
glUniform1i ( glGetUniformLocation ( program, "backgroundTex" ), 2 );
glMatrixMode ( GL_PROJECTION );
glLoadIdentity();
glOrtho ( 0,1,0,1,-1,1 );
glMatrixMode ( GL_MODELVIEW );
glLoadIdentity();
while ( glfwGetWindowParam ( GLFW_OPENED ) ) {
glClear ( GL_COLOR_BUFFER_BIT );
glBegin ( GL_QUADS );
glColor4f ( 1,1,1,1 );
glTexCoord2f ( 0,0 );
glVertex2f ( 0,0 );
glTexCoord2f ( 1,0 );
glVertex2f ( 1,0 );
glTexCoord2f ( 1,1 );
glVertex2f ( 1,1 );
glTexCoord2f ( 0,1 );
glVertex2f ( 0,1 );
glEnd();
glfwSwapBuffers();
}
return 0;
}
GLuint loadImage ( const char* filename ) {
GLuint tex = 0;
ILuint texid;
ilGenImages ( 1, &texid );
ilBindImage ( texid );
bool success = ilLoadImage ( ( const ILstring ) filename );
if ( success ) { /* If no error occured: */
success = ilConvertImage ( IL_RGB, IL_UNSIGNED_BYTE );
if ( !success ) {
cerr << "Failed to convert texture\n";
return 0;
}
glGenTextures ( 1, &tex );
glBindTexture ( GL_TEXTURE_2D, tex );
glTexParameteri ( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri ( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexImage2D ( GL_TEXTURE_2D, 0, ilGetInteger ( IL_IMAGE_BPP ), ilGetInteger ( IL_IMAGE_WIDTH ),
ilGetInteger ( IL_IMAGE_HEIGHT ), 0, ilGetInteger ( IL_IMAGE_FORMAT ), GL_UNSIGNED_BYTE,
ilGetData() );
} else {
cerr << "Failed to load texture\n";
return 0;
}
ilDeleteImages ( 1, &texid );
return tex;
}
const char *fsrc = "\
uniform sampler2D foregroundTex, maskTex, backgroundTex; \
\
void main() {\
float mask;\
vec4 result;\
mask = texture2D(maskTex,gl_TexCoord[0].st).r;\
result = texture2D(backgroundTex,gl_TexCoord[0].st) * (1.0f - mask);\
result = result + texture2D(foregroundTex,gl_TexCoord[0].st) * mask;\
result.a = 1.0;\
gl_FragColor = vec4(result);\
}";
const char *vsrc = "\
void main() { \
gl_Position = gl_ProjectionMatrix * gl_ModelViewMatrix * gl_Vertex;\
gl_TexCoord[0] = gl_MultiTexCoord0;\
}";