0
\$\begingroup\$

I'm having an issue implementing an HQX filter that I found in the LibRetro GitHub repository.

  • LUT texture was obtained here
  • Shader file was obtained here (I converted it to compile in MonoGame with SM 5)

This is what I'm getting:

Actual

And this is what I'm expecting:

Expected

This is the relevant code:

// loading the effect
var effect = base.Content.Load<Effect>(@"Effects\hqx");
var lutTexture = base.Content.Load<Texture2D>(@"Effects\hq4x");
var projection = Matrix.CreateOrthographicOffCenter(0, 720, 480, 0, 0, 1);
var halfPixelOffset = Matrix.CreateTranslation(0.5f, 0.5f, 0);

effect.Parameters["modelViewProj"].SetValue(halfPixelOffset * projection);
effect.Parameters["texture_size"].SetValue(new Vector2(240f, 160f));
effect.Parameters["LutTexture"].SetValue(lutTexture);

this.effect = effect;

Here is how I've configured the required LUT image:

LUT Configuration

Here is where I draw a pre-rendered texture using the effect:

// renderTarget is a 240x160 texture that has the image draw to it first
// the scaling is done by drawing the renderTarget at the higher resolution
spriteBatch.Begin(SpriteSortMode.Deferred, BlendState.NonPremultiplied,
                  SamplerState.PointClamp, null, null, this.Effect);
{
    spriteBatch.Draw(this.renderTarget, new Rectangle(0, 0, 720, 480), 
                     this.renderTarget.Bounds, Color.White);
}
spriteBatch.End();

This is the contents of the hqx.fx file:

#define SCALE 4

#define trY 48.0
#define trU 7.0
#define trV 6.0

static float3 yuv_threshold = float3(trY/255.0, trU/255.0, trV/255.0);

const static float3x3 yuv = float3x3(0.299, 0.587, 0.114, -0.169, -0.331, 0.5, 0.5, -0.419, -0.081);
const static float3 yuv_offset = float3(0, 0.5, 0.5);

bool diff(float3 yuv1, float3 yuv2) {
    bool3 res = abs((yuv1 + yuv_offset) - (yuv2 + yuv_offset)) > yuv_threshold;
    return res.x || res.y || res.z;
}

struct out_vertex {
    float4 position : SV_POSITION;
    float4 color    : COLOR;
    float2 texCoord : TEXCOORD0;
    float4 t1       : TEXCOORD1;
    float4 t2       : TEXCOORD2;
    float4 t3       : TEXCOORD3;
};

sampler2D decal : register(s0);
sampler2D LUT : register(s1)
{ 
    Texture = (LutTexture);
    Filter = Linear;  
    AddressU = clamp;
    AddressV = clamp;
};

float2 texture_size;
float4x4 modelViewProj;

out_vertex main_vertex
(
    float4 position : SV_POSITION,
    float4 color    : COLOR,
    float2 texCoord : TEXCOORD0
)
{
    out_vertex OUT;

    OUT.position = mul(position, modelViewProj);
    OUT.color = color;

    float2 ps = 1.0/texture_size;
    float dx = ps.x;
    float dy = ps.y;

    OUT.texCoord = texCoord;
    OUT.t1 = texCoord.xxxy + float4(-dx, 0, dx, -dy); //  w1 | w2 | w3
    OUT.t2 = texCoord.xxxy + float4(-dx, 0, dx,   0); //  w4 | w5 | w6
    OUT.t3 = texCoord.xxxy + float4(-dx, 0, dx,  dy); //  w7 | w8 | w9

    return OUT;
}
float4 main_fragment(in out_vertex VAR) : COLOR
{
    float2 fp = frac(VAR.texCoord*texture_size);
    float2 quad = sign(-0.5 + fp);
    float2 ps = 1.0/texture_size;

    float dx = ps.x;
    float dy = ps.y;
    float3 p1 = tex2D(decal, VAR.texCoord).rgb;
    float3 p2 = tex2D(decal, VAR.texCoord + float2(dx, dy) * quad).rgb;
    float3 p3 = tex2D(decal, VAR.texCoord + float2(dx, 0) * quad).rgb;
    float3 p4 = tex2D(decal, VAR.texCoord + float2(0, dy) * quad).rgb;
    float4x3 pixels = float4x3(p1, p2, p3, p4);

    float3 w1  = mul(yuv, tex2D(decal, VAR.t1.xw).rgb);
    float3 w2  = mul(yuv, tex2D(decal, VAR.t1.yw).rgb);
    float3 w3  = mul(yuv, tex2D(decal, VAR.t1.zw).rgb);

    float3 w4  = mul(yuv, tex2D(decal, VAR.t2.xw).rgb);
    float3 w5  = mul(yuv, p1);
    float3 w6  = mul(yuv, tex2D(decal, VAR.t2.zw).rgb);

    float3 w7  = mul(yuv, tex2D(decal, VAR.t3.xw).rgb);
    float3 w8  = mul(yuv, tex2D(decal, VAR.t3.yw).rgb);
    float3 w9  = mul(yuv, tex2D(decal, VAR.t3.zw).rgb);

    bool3x3 pattern = bool3x3(diff(w5, w1), diff(w5, w2), diff(w5, w3),
                              diff(w5, w4), false       , diff(w5, w6),
                              diff(w5, w7), diff(w5, w8), diff(w5, w9));
    bool4 cross = bool4(diff(w4, w2), diff(w2, w6), diff(w8, w4), diff(w6, w8));

    float2 index;
    index.x = dot(pattern[0], float3(1, 2, 4)) +
              dot(pattern[1], float3(8, 0, 16)) +
              dot(pattern[2], float3(32, 64, 128));
    index.y = dot(cross, float4(1, 2, 4, 8)) * (SCALE * SCALE) +
              dot(floor(fp * SCALE), float2(1, SCALE));

    float2 step = 1.0 / float2(256.0, 16.0 * (SCALE * SCALE));
    float2 offset = step / 2.0;
    float4 weights = tex2D(LUT, index * step + offset);
    float sum = dot(weights, float4(1,1,1,1));
    float3 res = mul(transpose(pixels), weights / sum);

    return float4(res, 1.0);
}

technique T0
{
    pass P0
    {
        VertexShader = compile vs_5_0 main_vertex();
        PixelShader = compile ps_5_0 main_fragment();
    }
}

Any help is greatly appreciated. If there are any other details you'd like me to provide please let me know.

\$\endgroup\$
0
\$\begingroup\$

When configuring the LUT texture you must set the properties in the pipeline as follows:

GenerateMipmaps  = false
PremultiplyAlpha = false

// set this to false as it is redundant anyway (the scale 4 texture is 256x256)
ResizeToPowerOfTwo = false
\$\endgroup\$
  • \$\begingroup\$ Glad you figured it out. Please remember to mark this as the accepted answer (the grey checkmark) after the requisite waiting period (2 days, I think). \$\endgroup\$ – ashes999 Oct 8 '15 at 19:26

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.