I'm at wits end here.
I recently revisited an "almost working" directional lighting / shadow mapping shader with the intent that maybe this time I could finally get it figured out.
I eventualy noticed that trying to override the camera view matrices did not seem to have the desired effect and when pondering that I discovered I didn't even have the latest DBP version installed (I recently reformatted my computer, so I guess I merely forgot to update it).
Now after I did upgrade to version 1.761 my shader has stopped working completely.
I cannot figure out why on earth this would happen; it takes several seconds to load and once it's loaded up my DBP project that utilizes it renders nothing at all for the objects which have it applied. When testing it with the DarkSHADER editor it takes even longer to compile the shader and eventually gives up, telling me the DBPro window isn't responding. A few seconds later it outright crashes.
I haven't changed the shader one bit since it was working with my old DBP version (from 2010 something I think, might have been version 1.6x). Now I have went over every line, trying to remove things to only have the essentials left but still to no avail. As far as I can tell it crashes because of some overload in the DirLightPS pixel shader, but compiling it with fxc says it should only use ~380 instruction slots (vertex / pixel shader 3.0 should allow for at least 512).
If anybody feels like taking a look at my shader, be my guest.
I've removed some half-unfinished point light functionality from it to make it more concrete if the ordering of instructions looks odd as a result of that.
// Untweakables (automatically set by the engine for each frame)
float4x4 WorldViewProj : WorldViewProjection;
float4x4 World : World;
float4 Eye : CameraPosition;
// Tweakables (can be set manually at arbitrary intervals)
float AmbientIntensity = 0.2f;
float4 AmbientColour = {0.25f, 0.25f, 0.25f, 1.0f};
float SpecularPower = 80.0f; // TODO: Use specular map
float4 DirectionalLightDir = {0.5f, -1.0f, 0.0f, 1.0f};
float4 DirectionalLightColour = {1.0f, 1.0f, 1.0f, 1.0f};
float4x4 DirectionalLightView;
float4x4 DirectionalLightProj;
float FarClipDistance = 1024.0f;
float ShadowBias = 0.0000039f;
float DepthMapTexelSize = 1.0 / 64.0f; // (1 / shadow map side)
// Textures and samplers
texture DiffuseMap < string Name = ""; >;
texture NormalMap < string Name = ""; >;
texture DepthMap < string Name = ""; >;
sampler DiffuseSampler = sampler_state {
Texture = < DiffuseMap >;
AddressU = Wrap;
AddressV = Wrap;
MinFilter = Linear;
MagFilter = Linear;
MipFilter = Linear;
};
sampler NormalSampler = sampler_state {
Texture = < NormalMap >;
AddressU = Wrap;
AddressV = Wrap;
MinFilter = Linear;
MagFilter = Linear;
MipFilter = Linear;
};
sampler DepthSampler = sampler_state {
Texture = < DiffuseMap >;
AddressU = Clamp;
AddressV = Clamp;
MinFilter = None;
MagFilter = None;
MipFilter = None;
};
// Vertex input data (supplied by the engine)
struct vs_in {
float4 pos : POSITION;
float3 normal : NORMAL;
float2 uv : TEXCOORD0;
float3 tangent : TANGENT;
};
struct vs_depth_in {
float4 pos : POSITION;
float2 uv : TEXCOORD0;
};
// Output from the vertex shader; input to the pixel shader
struct vs_out {
float4 pos : POSITION;
float3 normal : TEXCOORD0;
float2 uv : TEXCOORD1;
float4 lightPos : TEXCOORD2;
float4 worldPos : TEXCOORD3;
float3 view : TEXCOORD4;
float3 tangent : TANGENT;
};
struct vs_depth_out {
float4 pos : POSITION;
float2 uv : TEXCOORD0;
float4 wPos : TEXCOORD1;
};
/**
* Helper functions
**/
float luminance(float3 rgb) {
return rgb.r * 0.3 + rgb.g * 0.59 + rgb.b * 0.11;
}
/**
* Vertex shaders
**/
vs_depth_out DepthVS(vs_depth_in IN) {
vs_depth_out OUT;
// Ensure that the position vector uses 4 units (and that the fourth isn't random unset garbage) for
// proper matrix calculations
IN.pos.w = 1.0f;
// Calculate the vertex position against the light view projection matrices
OUT.pos = mul(mul(mul(IN.pos, World), DirectionalLightView), DirectionalLightProj);
// Store the actual WVP position so that it can be used in the pixel shader (since the POSITION semantic seems to impose
// certain access right restrictions on its value)
OUT.wPos = OUT.pos;
// Forward texture coordinates
OUT.uv = IN.uv;
return OUT;
}
vs_out DirLightVS(vs_in IN) {
vs_out OUT;
// Transform position from local to projection space
IN.pos.w = 1.0f;
OUT.pos = mul(IN.pos, WorldViewProj);
// We also need the world position in the pixel shader for determining distances
OUT.worldPos = mul(IN.pos, World);
// Compute a 3x3 matrix to transform world space to tangent ("texture") space
float3x3 worldToTangentSpace;
worldToTangentSpace[0] = mul(IN.tangent, World);
worldToTangentSpace[1] = mul(cross(IN.tangent, IN.normal), World);
worldToTangentSpace[2] = mul(IN.normal, World);
// Forward the tangent to the pixel shader for world->texture space transformations and normal mapping
OUT.tangent = IN.tangent;
// Compute position in the directional light's world view projection space
OUT.lightPos = mul(IN.pos, mul(World, mul(DirectionalLightView, DirectionalLightProj)));
// Transform normal
OUT.normal = mul(IN.normal, (float3x3)World); // N
float3 posWorld = normalize(mul(IN.pos, World));
OUT.view = mul(worldToTangentSpace, Eye - posWorld); // V
// Forward texture coordinates
OUT.uv = IN.uv;
return OUT;
}
/**
* Pixel shaders
**/
float4 DepthPS(vs_depth_out IN) : COLOR {
float depth = IN.wPos.z / IN.wPos.w;
return float4(depth, depth, depth, 1.0f);
}
float4 DirLightPS(vs_out IN) : COLOR {
// Compute world to tangent space transform matrix; this is supposedly
// dependant on the real surface normals (as stiored in the mesh) and
// should be done before we replace those values with normals read from
// the normal map
float3x3 worldToTangentSpace;
worldToTangentSpace[0] = mul(IN.tangent, World);
worldToTangentSpace[1] = mul(cross(IN.tangent, IN.normal), World);
worldToTangentSpace[2] = mul(IN.normal, World);
// Sample normals from the associated normal map rather than use the
// mesh-defined ones
IN.normal = normalize(2 * (tex2D(NormalSampler, IN.uv) - 0.5f));
// Re-homogenize after the automatically applied interpolation for TEXCOORD registers
IN.lightPos.xyzw /= IN.lightPos.w;
// Is this position inside the directional light's frustum?
// If not it is not illuminated by it.
float shadowed = 0.0f;
float shadowFactor = IN.lightPos.x < -1.0f || IN.lightPos.x > 1.0f ||
IN.lightPos.y < -1.0f || IN.lightPos.y > 1.0f ||
IN.lightPos.z < -1.0f || IN.lightPos.z > 1.0f;
if(shadowFactor == 0.0f) {
// Transform clip space coordinates to texture space coordinates ([-1..1] to [0..1])
IN.lightPos.x = IN.lightPos.x / 2 + 0.5f;
IN.lightPos.y = IN.lightPos.y / 2 + 0.5f;
// Apply shadow map bias (don't compare exact depths to alleviate precision issues)
IN.lightPos.z -= ShadowBias;
// If our current fragment's clip space z-coordinate is greater than
// the shadow maps's, the fragment (pixel) is shadowed.
// Use percentage closer filtering (PCF) to smooth the shadows a bit
float2 texcoord[25];
for(int x = 0; x < 5; x++) {
for(int y = 0; y < 5; y++) {
texcoord[(y * 5) + x] = IN.lightPos.xy + float2(DepthMapTexelSize * (x - 2), DepthMapTexelSize * (y - 2));
}
}
for(int p = 0; p < 25; p++)
shadowed += tex2D(DepthSampler, texcoord[p]).r < IN.lightPos.z ? 1.0f : 0.0f;
shadowFactor = shadowed / 25;
}
float3 colour = AmbientColour.xyz; // Initiate the final colour to the ambient colour
float3 specCol = float3(0.0f, 0.0f, 0.0f); // Default specular colour / intensity for directional light
// Apply directional light influence (assumed to always be present to some extent)
float3 lightDir = normalize(-DirectionalLightDir);
// Calculate diffuse directional light
colour += abs(dot(lightDir, IN.normal)) * DirectionalLightColour * (1.0f - shadowFactor);
// Calculate specular directional light
float3 camera = mul(Eye, World);
float3 V = normalize(camera - IN.worldPos);
float3 H = normalize(-DirectionalLightDir + V);
specCol = pow(saturate(dot(IN.normal, H)), 4) * DirectionalLightColour * (1.0 - shadowFactor);
// Diffuse colour as sampled from the model's diffuse texture at this fragment / pixel
float4 texCol = tex2D(DiffuseSampler, IN.uv);
float3 outputColour = colour * texCol.xyz + specCol;
//return float4(colour * texCol.xyz + specCol, texCol.w); // NOTE: THIS CAUSES CRASHES??
//return float4(clamp(outputColour.r, 0.0f, 1.0f), clamp(outputColour.g, 0.0f, 1.0f), clamp(outputColour.b, 0.0f, 1.0f), texCol.w);
//return float4(0.0f, 1.0f, 0.0f, 1.0f); // This still compiles and looks apropriate
return float4(colour.xyz * texCol.xyz + specCol, texCol.w);
// UPDATE: Adding specCol causes very slow loading.
// Using colour instead of AmbientColour.xyz as the multiplier causes an outright crash
}
technique DirectionalLighting {
pass p0 {
VertexShader = compile vs_3_0 DirLightVS();
PixelShader = compile ps_3_0 DirLightPS();
}
}
technique RenderDepthMap {
pass p0 {
ColorWriteEnable = Red;
VertexShader = compile vs_3_0 DepthVS();
PixelShader = compile ps_3_0 DepthPS();
}
}
Finally, any ideas on what might be causing this would be greatly appreciated. Googling hasn't yielded a single similar thing so far.
Edit: Tried to make the tabs in the code block look more coherent.
"Why do programmers get Halloween and Christmas mixed up?" Because Oct(31) = Dec(25)