Source available on GitHub.

Alpha Masks

Using an alpha mask is like magic. Before I knew about them, any effect that used the technique was super confusing to figure out. The basic idea is that you control how much of a texture is shown by cutting off all pixels with an alpha value less (or greater) than the value you pass in. That probably wasn't the easiest explanation to understand, so here's an example:


Look at the dash cooldown, the circle filling up on the ground. This is an alpha mask! There is no easy way to make a texture fade in from a circular pattern with default sprite rendering stuff, so we used this texture:


The black background is just there so you can see the white texture. To render this, we pass in the percent that the cooldown has progressed, from 0-1. When the cooldown is at, say, .35f, only the pixels in the texture that have an alpha value >= .35f are rendered. The alpha is ignored in the final color, the actual texture is a solid color. But we can use this to gradually ease in the cooldown texture in a circular pattern.

You can use this technique to fade textures in/out in any pattern you want. As long as you can photoshop the alpha gradient for the pattern, you can make an alpha mask for it.




AlphaCutoffBillboard.fx

The HLSL code for cutting off pixels of a billboard based on the alpha channel.

 
/*
 * A shader for cutting off pixels that don't have the required alpha values.
 */
 
#include "Macros.fxh"
 
DECLARE_TEXTURE(Texture, 1);
DECLARE_TEXTURE(DepthMap, 2);
 
cbuffer Parameters : register(b0)
{
	float4x4 View;
	float4x4 Projection;
	float3 colorTint;
	float alpha;
	float alphaCutoff;
	bool invertCutoff;
}
 
 
struct VSIn
{
    float4 Position : SV_Position;
    float2 TexCoord : TEXCOORD0;
};
 
struct VSOut
{
    float4 Position  : SV_Position;
    float2 TexCoord  : TEXCOORD0;
	float4 ScreenPos : TEXCOORD1;
};
 
VSOut VS(VSIn input)
{
    VSOut output;
 
	output.Position = mul(mul(input.Position, View), Projection);
 
	//also output the position as a texcoord, so that it 
	//is interpolated correctly for figuring out position on the depth map
	output.ScreenPos = output.Position;
	output.TexCoord = input.TexCoord;
 
    return output;
}
 
float4 PS(VSOut input) : SV_Target
{
	float billboardDepth = input.ScreenPos.z / input.ScreenPos.w;
 
	//divide by homogenous coordinate
	input.ScreenPos.xy /= input.ScreenPos.w;
	//transform from [-1, 1] to [0, 1]
	float2 texCoord = .5f * (float2(input.ScreenPos.x, -input.ScreenPos.y) + 1);
	float sceneDepth = SAMPLE_TEXTURE(DepthMap, texCoord);
	if(billboardDepth > sceneDepth)
	{
		discard;
	}
 
 
	//THE ACTUAL ALPHA CUTOFF PART
	float4 color = SAMPLE_TEXTURE(Texture, input.TexCoord);
 
	//if transparent / black, discard (not part of texture)
	if(color.a == 0)
	{
		discard;
	}
 
	//otherwise, use cut off texture at given alpha value
	if(invertCutoff)
	{
		if(color.a <= alphaCutoff)
		{
			discard;
		}
	}
	else
	{
		if(color.a > alphaCutoff)
		{
			discard;
		}
	}
 
	color = color * float4(colorTint, 1);
	//finally, use custom alpha (the texture's gradient is not shown in final product)
	color.a = alpha;
	return color;
}
 
TECHNIQUE(Billboard, VS, PS);
 
 






But wait, there's more! We can take this a step further, and apply this technique to 3D models! Since we are usually mapping textures to models and processing them pixel-by-pixel, that means we could theoretically look at the alpha value for that pixel, and do a similar kind of comparison.

In Metagalactic Blitz, Veronica constructs turrets with this technique. First a model with a wireframe texture is cut in, and then it is cut out while the real model is cut in. At the end of it, both are replaced by the real animated model.



This uses the exact same theory, except I passed in a separate texture for the alpha mask this time so that I could separate the real texture and the alpha mask. The only thing is, you have to plan ahead for this kind of model. It requires a second set of texture coordinates in the vertex declaration so that the alpha mask can be mapped to it. (note the "vin.TexCoord2" in the vertex shader).


DeferredAlphaMaskModel.fx

The HLSL code for cutting off pixels of a model based on the alpha channel.

 
//you can find these files in the full GitHub download
#include "Macros.fxh"
#include "NormalEncoding.fxh"
#include "Structures.fxh"
 
Texture2D<float4> Texture : register(t0);
sampler TextureSampler = sampler_state
{
   Texture = <Texture>;
   MinFilter = Linear;
   MagFilter = Linear;
   MipFilter = Linear;
   AddressU  = Wrap;
   AddressV  = Wrap;
};
Texture2D<float4> AlphaMaskTexture : register(t1);
sampler AlphaMaskTextureSampler = sampler_state
{
   Texture = <AlphaMaskTexture>;
   MinFilter = Linear;
   MagFilter = Linear;
   MipFilter = Linear;
   AddressU  = Wrap;
   AddressV  = Wrap;
};
 
cbuffer Parameters : register(b0)
{
	float3 colorTint;
	float tintLerp;
	bool hasOutline;
	float alphaCutoff;
	bool invertCutoff;
	float4x4 World;
	float4x4 WorldViewProj;
}
 
ToonVSMskOutput VSDeferred(VSInputNmTxMsk vin)
{
	ToonVSMskOutput output;
 
	output.TexCoord = vin.TexCoord1;
	output.PositionPS = mul(vin.Position, WorldViewProj);
	output.Normal = mul(vin.Normal, World);
	output.Depth.xy = output.PositionPS.zw;
	output.AlphaMaskTexCoord = vin.TexCoord2;
 
 
    return output;
}
 
GBufferPSOutput PSDeferred(ToonVSMskOutput pin) : SV_Target0
{
	GBufferPSOutput output;
 
	//THE ALPHA CUTOFF PART
	float4 alphaMask = AlphaMaskTexture.Sample(AlphaMaskTextureSampler, pin.AlphaMaskTexCoord);
	if(invertCutoff && alphaMask.a <= alphaCutoff || !invertCutoff && alphaMask.a > alphaCutoff)
	{
		discard;
	}
 
	//just generic model rendering
	float4 tex = Texture.Sample(TextureSampler, pin.TexCoord);
	output.Color.rgb = lerp(tex, colorTint, tintLerp);
	output.Color.a = (hasOutline? 1 : 0);
 
	output.Normal.xy = encodeNormals(normalize(pin.Normal));
	output.Normal.z = 0;//specular intensity
	output.Normal.w = 0;//specular power
 
	output.Depth = pin.Depth.x / pin.Depth.y;
	return output;
}
 
TECHNIQUE(Deferred, VSDeferred, PSDeferred);