0
\$\begingroup\$

I know this question's been asked many times before, but I can't seem to get sprite masking to work in XNA/Monogame no matter if I try the stencil method, or the shader method.

Considering I'm having partial luck with the shader method, I feel I'll have the most luck with this route. Though it's not consistently sampling the same region of the texture, nor is it sampling the sprite I want to, nor is it sampling 100% correctly since the rest of the sprites are being rendered with partial transparency; it's still further than I've gotten with stenciling.

However, even with regards to all the examples I've seen online, my problem is slightly different. Basically, my sprites are all unpacked from a file that contains a list of indexed sprites into into 2048x2048 square textures. The mask sprite also goes into this same list, so it could be part of the same texture as one of the character sprites at any arbitrary region. What this means is that the mask sprite does not have its own texture. This complicates things. Additionally, I want to be able to place the mask spot at any region of the main sprite, whether it's outside or inside the region, and get the cut I expect. Furthermore, the mask sprite need not be the same dimensions as the main sprite.

Thus I could have this, a 66x121px image file:

enter image description here

Being masked with this, a 45x45 image file:

enter image description here

And the result would be something like: the black triangle defines what gets masked out; anything white or outside of the mask region gets kept.

Once again, these sprites are within the region of (separate) 2048x2048 texture.

This is my relevant HLSL code:

sampler TexSampler : register(s0) = sampler_state { MinFilter = Point; MagFilter = Point; };
sampler PalSampler : register(s1) = sampler_state { MinFilter = Point; MagFilter = Point; };
sampler MaskSampler : register(s2) = sampler_state { addressU = Clamp; addressV = Clamp; };

uniform int PalTextureSize : register(c0);

uniform float2 palCoords;

uniform float3 add;
uniform float3 mult;
uniform bool invertAll;
uniform float colorbal;
uniform float src_alphamul;

float2 drawSpritePos;
float2 drawSpriteSize;
float2 maskSpritePos;
float2 maskSpriteSize;

// ... pallete application code...

//
// Applies a mask to an image. The parameter is the main texture coordinate.
//
void Mask(float2 texCoord, inout float4 color)
{
    // Where does the sprite get drawn?
    //float2 spriteStart = drawSpritePos + drawSpriteSize*texCoord;
    //float2 mappedCoord = (maskCoord * maskSpriteSize) + maskSpritePos;

    // Get the necessary values from each sampler
    //float alphaValue = tex2D(MaskSampler, texCoord * maskCoord).r; // this does something, not sure what though
    //color.a = alphaValue;
    float alphaValue = 0.0;
    if (any(tex2D(MaskSampler, texCoord).rgb)) // A MASK THAT LOOKS LIKE WHAT I WANT OMG 
        alphaValue = 1.0;

    color *= alphaValue;
}

//
// Shader converting 8-bit indexed color to RGBA
//
float4 Main(float2 texCoord : TEXCOORD0, float2 maskCoord : TEXCOORD2) : COLOR0
{
    float4 color = NearestNeighbor(texCoord);
    PalFX(color);
    Mask(maskCoord * texCoord, color);
    return color;
}

technique ApplyPal
{
    pass Pass0
    {
        PixelShader = compile ps_3_0 Main();
    }
}

And this is my Draw() method in my Entity class:

    public virtual void Draw(SpriteBatch spriteBatch, Effect spriteEffect, Texture2D[] plTextures, GameTime gameTime)
    {
        Sprite plSprite = null;
        bool beginCalled = false;
        Collision clsn = currCollision;
        // Just TRY to draw in case someone tries to draw an invalid sprite or something
        try
        {
            plSprite = Sprites.Select((s) => s).Where(s => s.ID == Anim.CurrElem.SpriteID).First();
            var maskSprite = Sprites.Select((s) => s).Where(s => s.ID == new ID(54321, 0)).First();
            Texture2D p1Tex = plTextures[plSprite.TextureIndex];
            //spriteEffect.Parameters["src_alphamul"].SetValue(1.0F);
            spriteEffect.Parameters["colorbal"].SetValue(1.0F);
            spriteEffect.Parameters["mult"].SetValue(new Vector3(1.0F, 1.0F, 1.0F));
            textureRegion = plSprite.TextureRegion;

            int palIndex = plSprite.PaletteIndex;
            if (paletteRemappings.ContainsKey(palIndex))
                palIndex = paletteRemappings[palIndex];

            spriteEffect.Parameters["MaskSampler"].SetValue(plTextures[maskSprite.TextureIndex]);
            Vector2 maskPos = maskSprite.TextureRegion.Location.ToVector2() / maskSprite.TextureRegion.Size.ToVector2();

            if (palIndex > -1)
            {
                spriteBatch.Begin(effect: spriteEffect, samplerState: SamplerState.PointClamp, rasterizerState: RasterizerState.CullNone);
                beginCalled = true;
                foreach (var pass in spriteEffect.CurrentTechnique.Passes)
                {
                    pass.Apply();
                    spriteEffect.Parameters["palCoords"].SetValue(new Vector2((palIndex / 4) % 1024, (palIndex / 4) / 1024));
                    spriteBatch.Draw(p1Tex, Pos + new Vector2(0,StageHeight), textureRegion, Color.White, Anim.CurrElem.Rotation, plSprite.Axis, Scale * Anim.CurrElem.Scale * new Vector2(facing, 1), SpriteEffects.None, LayerDepth);
                    //spriteBatch.Draw(plTextures[maskSprite.TextureIndex], Pos + new Vector2(0,StageHeight), maskSprite.TextureRegion, Color.White, 0, maskSprite.Axis, Scale * new Vector2(facing, 1), SpriteEffects.None, LayerDepth);

                }
            }
            else
            {
                spriteBatch.Begin(samplerState: SamplerState.PointClamp, blendState: BlendState.NonPremultiplied, rasterizerState: RasterizerState.CullNone);
                beginCalled = true;
                spriteBatch.Draw(p1Tex, StagePos, plSprite.TextureRegion, Color.White, Anim.CurrElem.Rotation, plSprite.Axis, Scale * Anim.CurrElem.Scale * new Vector2(facing, 1), SpriteEffects.None, LayerDepth);
            }

            if (beginCalled)
            {
                spriteBatch.End();
                beginCalled = false;
            }
        }
        catch (Exception)
        {
            if (beginCalled)
                spriteBatch.End();
        }
    }

It'd hard for me to debug, though, since this is HLSL being translated into GLSL internally since this is an OpenGL MonoGame project in VS2017. I'm using NVidia's Nsight debugger which is awesome, but it doesn't help me much in the way of debugging since I have no idea what I'm doing.

I've been stuck on this problem for 36 straight hours, so any help would be greatly appreciated!

\$\endgroup\$
0
\$\begingroup\$

After much trial and error, I did it! Didn't have to use stencils, either (which didn't seem to work anyway).

Basically, using a non-indexed mask sprite, I have to pass in these parameters:

spriteEffect.Parameters["MaskSampler"].SetValue(plTextures[maskSprite.TextureIndex]);
spriteEffect.Parameters["drawSpritePos"].SetValue(plSprite.TextureRegion.Location.ToVector2());
spriteEffect.Parameters["maskSpritePos"].SetValue(maskSprite.TextureRegion.Location.ToVector2());
spriteEffect.Parameters["maskSpriteSize"].SetValue(maskSprite.TextureRegion.Size.ToVector2());

drawspritePos is the location in the player texture to start drawing from. maskSpritePos and maskSpriteSize are similar in that regards, but for the mask sprite texture.

The trickiest part was the shader, where the final mixing is done:

float4 Main(float2 texCoord : TEXCOORD0) : COLOR0
{
    // Return of the mask
    float2 texIndex = texCoord * 2048.0;
    float2 maskIndex = texIndex - drawSpritePos;
    float2 maskPixel = (maskSpritePos + maskIndex) / 2048.0;
    float4 mask = tex2D(MaskSampler, maskPixel);

    // Convert indexed to RGBA
    float4 color = NearestNeighbor(TexSampler, texCoord, normPalCoords);
    PalFX(color);

    // Within the mask texture region?
    if ((maskPixel*2048.0 - maskSpritePos).x < maskSpriteSize.x && (maskPixel*2048.0 - maskSpritePos).y < maskSpriteSize.y)
        color *= mask.a;

    return color;
}

The "2048" I need to make a constant, but where it comes from is the texture size I use for sprites since the sprites themselves are always contained in a region within a 2048x2048 textures.

The drawing is pretty simple. No need to draw the mask, only the sprite to be masked.

if (palIndex > -1)
{
    spriteBatch.Begin(effect: spriteEffect, samplerState: SamplerState.PointClamp, rasterizerState: RasterizerState.CullNone, depthStencilState: s2);
    beginCalled = true;
    spriteEffect.CurrentTechnique.Passes[0].Apply();
    palIndex = plSprite.PaletteIndex;
    if (paletteRemappings.ContainsKey(palIndex))
        palTexIndex = paletteRemappings[palIndex];
    else
        palTexIndex = Palettes[palIndex].StartIndex;

    // Divide by 4 because of RGBA
    spriteEffect.Parameters["normPalCoords"].SetValue(new Vector2((palTexIndex / 4) % 1024, ((palTexIndex) / 4.0F) / 1024.0F));
    spriteBatch.Draw(p1Tex, Pos + new Vector2(0,StageHeight), textureRegion, Color.White, Anim.CurrElem.Rotation, plSprite.Axis, Scale * Anim.CurrElem.Scale * new Vector2(facing, 1), SpriteEffects.None, LayerDepth);
}
// 24 or 32 bit sprite
else
{
    spriteBatch.Begin(samplerState: SamplerState.PointClamp, blendState: BlendState.NonPremultiplied, rasterizerState: RasterizerState.CullNone);
    beginCalled = true;
    spriteBatch.Draw(p1Tex, StagePos, plSprite.TextureRegion, Color.White, Anim.CurrElem.Rotation, plSprite.Axis, Scale * Anim.CurrElem.Scale * new Vector2(facing, 1), SpriteEffects.None, LayerDepth);
}
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.