Changing outline colour of toon shader?

Author

Message

TheComet

18

Years of Service

User Offline

Joined: 18th Oct 2007

Location: I`m under ur bridge eating ur goatz.

Posted: 20th Dec 2011 20:49

Link

How can I change the colour of the outline in this toon shader to something variable? Here's the shader:

//====================================================
// Toon shader
//====================================================
// By EVOLVED
// www.evolved-software.com
//====================================================

//--------------
// un-tweaks
//--------------
   matrix WorldVP:WorldViewProjection; 
   matrix World:World;   
   matrix ViewInv:ViewInverse;

//--------------
// tweaks
//--------------
   float3 LightPosition={0.0f,50.0f,-250.0f};  
   float EdgeDis=0.4f;

//--------------
// Textures
//--------------
   texture BaseTX <string Name="";>;	
   sampler2D Base = sampler_state 
    {
 	texture = <BaseTX>;
    };
   texture ShadeTX <string Name="";>;	
   sampler2D Shade = sampler_state 
      {
 	texture = <ShadeTX>;
	MagFilter=None;
	MinFilter=None;
	MipFilter=None;	
      };

//--------------
// structs
//--------------
   struct input
      {
    	float4 Pos : POSITION ; 
    	float2 UV : TEXCOORD ; 
 	float3 Normal : NORMAL;  
      };
   struct output
      {
    	float4 OPos : POSITION ; 
    	float2 uv : TEXCOORD0 ; 
     	float2 Shade : TEXCOORD1 ;
      };
   struct output_Edge
      {
    	float4 OPos : POSITION ;
     	float4 Edge : COLOR0 ; 
      };

//--------------
// vertex shader
//--------------
   output VS(input IN) 
     {
    	output OUT;
    	OUT.OPos=mul(IN.Pos,WorldVP);
	OUT.uv=IN.UV;
	float3 WNor=mul(IN.Normal,World);  
	       WNor=normalize(WNor);
	float3 WPos=mul(IN.Pos,World); 
  	float3 LightVec=LightPosition-WPos;
   	float3 LightDir=mul(LightVec,WNor)/2;
	OUT.Shade.xy=saturate(LightDir/length(LightVec));
  	return OUT;
     }
   output_Edge VS_Edge(input IN) 
     {
    	output_Edge OUT;
	float4 Vpos;
	Vpos.xyz=IN.Pos+(IN.Normal*EdgeDis);Vpos.w=IN.Pos.w;
    	OUT.OPos=mul(Vpos,WorldVP);
	OUT.Edge.xyzw=0;
  	return OUT;
     }

//--------------
// techniques 
//--------------
    technique Toon
      {
 	pass p0
      {		
 	vertexShader = compile vs_1_1 VS(); 
        Sampler[0] = <Base>;
  	ColorOp[0]   = MODULATE;
        ColorArg1[0] = TEXTURE;
        Sampler[1] = <Shade>;
  	ColorOp[1]   = MODULATE;
        ColorArg1[1] = TEXTURE;
      }
  	pass p1
      {		
 	vertexShader = compile vs_1_1 VS_Edge(); 
        CullMode = Cw;
        Sampler[0] = null;
        Sampler[1] = null;
      }
      }

+ Code Snippet

//====================================================
// Toon shader
//====================================================
// By EVOLVED
// www.evolved-software.com
//====================================================

//--------------
// un-tweaks
//--------------
   matrix WorldVP:WorldViewProjection; 
   matrix World:World;   
   matrix ViewInv:ViewInverse; 

//--------------
// tweaks
//--------------
   float3 LightPosition={0.0f,50.0f,-250.0f};  
   float EdgeDis=0.4f;

//--------------
// Textures
//--------------
   texture BaseTX <string Name="";>;	
   sampler2D Base = sampler_state 
    {
 	texture = <BaseTX>;
    };
   texture ShadeTX <string Name="";>;	
   sampler2D Shade = sampler_state 
      {
 	texture = <ShadeTX>;
	MagFilter=None;
	MinFilter=None;
	MipFilter=None;	
      };

//--------------
// structs
//--------------
   struct input
      {
    	float4 Pos : POSITION ; 
    	float2 UV : TEXCOORD ; 
 	float3 Normal : NORMAL;  
      };
   struct output
      {
    	float4 OPos : POSITION ; 
    	float2 uv : TEXCOORD0 ; 
     	float2 Shade : TEXCOORD1 ;
      };
   struct output_Edge
      {
    	float4 OPos : POSITION ;
     	float4 Edge : COLOR0 ; 
      };

//--------------
// vertex shader
//--------------
   output VS(input IN) 
     {
    	output OUT;
    	OUT.OPos=mul(IN.Pos,WorldVP);
	OUT.uv=IN.UV;
	float3 WNor=mul(IN.Normal,World);  
	       WNor=normalize(WNor);
	float3 WPos=mul(IN.Pos,World); 
  	float3 LightVec=LightPosition-WPos;
   	float3 LightDir=mul(LightVec,WNor)/2;
	OUT.Shade.xy=saturate(LightDir/length(LightVec));
  	return OUT;
     }
   output_Edge VS_Edge(input IN) 
     {
    	output_Edge OUT;
	float4 Vpos;
	Vpos.xyz=IN.Pos+(IN.Normal*EdgeDis);Vpos.w=IN.Pos.w;
    	OUT.OPos=mul(Vpos,WorldVP);
	OUT.Edge.xyzw=0;
  	return OUT;
     }

//--------------
// techniques 
//--------------
    technique Toon
      {
 	pass p0
      {		
 	vertexShader = compile vs_1_1 VS(); 
        Sampler[0] = <Base>;
  	ColorOp[0]   = MODULATE;
        ColorArg1[0] = TEXTURE;
        Sampler[1] = <Shade>;
  	ColorOp[1]   = MODULATE;
        ColorArg1[1] = TEXTURE;
      }
  	pass p1
      {		
 	vertexShader = compile vs_1_1 VS_Edge(); 
        CullMode = Cw;
        Sampler[0] = null;
        Sampler[1] = null;
      }
      }

Evolved's example is attached if you need it.

Thanks,

TheComet

Back to top

Profile PM Email Website

Green Gandalf

VIP Member

21

Years of Service

User Offline

Joined: 3rd Jan 2005

Playing: Malevolence:Sword of Ahkranox, Skyrim, Civ6.

Posted: 20th Dec 2011 22:56 Edited at: 20th Dec 2011 22:58

Link

Change this

+ Code Snippet

OUT.Edge.xyzw=0;

to use whatever colour you want. For example to get full red use:

+ Code Snippet

OUT.Edge.xyzw=EdgeColour;

and add the declaration of EdgeColour to the tweaks:

+ Code Snippet

//--------------
// tweaks
//--------------
   float3 LightPosition={0.0f,50.0f,-250.0f};  
   float EdgeDis=0.4f;
   float4 EdgeColour = {1.0, 0.0, 0.0, 1.0}; // i.e. {red, green, blue, alpha}

You can then change the edge colour in the usual way using

+ Code Snippet

set vector4 vecId, red#, green#, blue#, alpha#
set effect constant vector fxId, "EdgeColour", vecId

Remember that the colour values need to be floats in the range 0 to 1.

[Health warning: untested.

]

Back to top

Profile PM Email

Southside Games

16

Years of Service

User Offline

Joined: 1st Jan 2010

Location: Don\'t tell me what to do!

Posted: 20th Dec 2011 23:45

Link

I have a question, I want to change the line color from object to object, can I do that?

Back to top

Profile PM

TheComet

18

Years of Service

User Offline

Joined: 18th Oct 2007

Location: I`m under ur bridge eating ur goatz.

Posted: 20th Dec 2011 23:53

Link

Thanks a lot!

Wow shader code is still boggling my mind... I'm having a hard time wrapping my head around it.

TheComet

Back to top

Profile PM Email Website

TheComet

18

Years of Service

User Offline

Joined: 18th Oct 2007

Location: I`m under ur bridge eating ur goatz.

Posted: 21st Dec 2011 00:01

Link

Sorry for the double post, but this here is still black for me... Where did my logic go wrong?

//====================================================
// Toon shader
//====================================================
// By EVOLVED
// www.evolved-software.com
//====================================================

//--------------
// un-tweaks
//--------------
   matrix WorldVP:WorldViewProjection; 
   matrix World:World;   
   matrix ViewInv:ViewInverse;

//--------------
// tweaks
//--------------
   float3 LightPosition={0.0f,50.0f,-250.0f};  
   float EdgeDis=0.4f;
   float4 EdgeColor={1.0f,0.0f,0.0f,1.0f};

//--------------
// Textures
//--------------
   texture BaseTX <string Name="";>;	
   sampler2D Base = sampler_state 
    {
 	texture = <BaseTX>;
    };
   texture ShadeTX <string Name="";>;	
   sampler2D Shade = sampler_state 
      {
 	texture = <ShadeTX>;
	MagFilter=None;
	MinFilter=None;
	MipFilter=None;	
      };

//--------------
// structs
//--------------
   struct input
      {
    	float4 Pos : POSITION ; 
    	float2 UV : TEXCOORD ; 
 	float3 Normal : NORMAL;  
      };
   struct output
      {
    	float4 OPos : POSITION ; 
    	float2 uv : TEXCOORD0 ; 
     	float2 Shade : TEXCOORD1 ;
      };
   struct output_Edge
      {
    	float4 OPos : POSITION ;
     	float4 Edge : COLOR0 ; 
      };

//--------------
// vertex shader
//--------------
   output VS(input IN) 
     {
    	output OUT;
    	OUT.OPos=mul(IN.Pos,WorldVP);
	OUT.uv=IN.UV;
	float3 WNor=mul(IN.Normal,World);  
	       WNor=normalize(WNor);
	float3 WPos=mul(IN.Pos,World); 
  	float3 LightVec=LightPosition-WPos;
   	float3 LightDir=mul(LightVec,WNor)/2;
	OUT.Shade.xy=saturate(LightDir/length(LightVec));
  	return OUT;
     }
   output_Edge VS_Edge(input IN) 
     {
    	output_Edge OUT;
	float4 Vpos;
	Vpos.xyz=IN.Pos+(IN.Normal*EdgeDis);Vpos.w=IN.Pos.w;
    	OUT.OPos=mul(Vpos,WorldVP);
	OUT.Edge.xyzw=EdgeColor;
  	return OUT;
     }

//--------------
// techniques 
//--------------
    technique Toon
      {
 	pass p0
      {		
 	vertexShader = compile vs_1_1 VS(); 
        Sampler[0] = <Base>;
  	ColorOp[0]   = MODULATE;
        ColorArg1[0] = TEXTURE;
        Sampler[1] = <Shade>;
  	ColorOp[1]   = MODULATE;
        ColorArg1[1] = TEXTURE;
      }
  	pass p1
      {		
 	vertexShader = compile vs_1_1 VS_Edge(); 
        CullMode = Cw;
        Sampler[0] = null;
        Sampler[1] = null;
      }
      }

+ Code Snippet

//====================================================
// Toon shader
//====================================================
// By EVOLVED
// www.evolved-software.com
//====================================================

//--------------
// un-tweaks
//--------------
   matrix WorldVP:WorldViewProjection; 
   matrix World:World;   
   matrix ViewInv:ViewInverse; 

//--------------
// tweaks
//--------------
   float3 LightPosition={0.0f,50.0f,-250.0f};  
   float EdgeDis=0.4f;
   float4 EdgeColor={1.0f,0.0f,0.0f,1.0f};

//--------------
// Textures
//--------------
   texture BaseTX <string Name="";>;	
   sampler2D Base = sampler_state 
    {
 	texture = <BaseTX>;
    };
   texture ShadeTX <string Name="";>;	
   sampler2D Shade = sampler_state 
      {
 	texture = <ShadeTX>;
	MagFilter=None;
	MinFilter=None;
	MipFilter=None;	
      };

//--------------
// structs
//--------------
   struct input
      {
    	float4 Pos : POSITION ; 
    	float2 UV : TEXCOORD ; 
 	float3 Normal : NORMAL;  
      };
   struct output
      {
    	float4 OPos : POSITION ; 
    	float2 uv : TEXCOORD0 ; 
     	float2 Shade : TEXCOORD1 ;
      };
   struct output_Edge
      {
    	float4 OPos : POSITION ;
     	float4 Edge : COLOR0 ; 
      };

//--------------
// vertex shader
//--------------
   output VS(input IN) 
     {
    	output OUT;
    	OUT.OPos=mul(IN.Pos,WorldVP);
	OUT.uv=IN.UV;
	float3 WNor=mul(IN.Normal,World);  
	       WNor=normalize(WNor);
	float3 WPos=mul(IN.Pos,World); 
  	float3 LightVec=LightPosition-WPos;
   	float3 LightDir=mul(LightVec,WNor)/2;
	OUT.Shade.xy=saturate(LightDir/length(LightVec));
  	return OUT;
     }
   output_Edge VS_Edge(input IN) 
     {
    	output_Edge OUT;
	float4 Vpos;
	Vpos.xyz=IN.Pos+(IN.Normal*EdgeDis);Vpos.w=IN.Pos.w;
    	OUT.OPos=mul(Vpos,WorldVP);
	OUT.Edge.xyzw=EdgeColor;
  	return OUT;
     }

//--------------
// techniques 
//--------------
    technique Toon
      {
 	pass p0
      {		
 	vertexShader = compile vs_1_1 VS(); 
        Sampler[0] = <Base>;
  	ColorOp[0]   = MODULATE;
        ColorArg1[0] = TEXTURE;
        Sampler[1] = <Shade>;
  	ColorOp[1]   = MODULATE;
        ColorArg1[1] = TEXTURE;
      }
  	pass p1
      {		
 	vertexShader = compile vs_1_1 VS_Edge(); 
        CullMode = Cw;
        Sampler[0] = null;
        Sampler[1] = null;
      }
      }

TheComet

Back to top

Profile PM Email Website

Green Gandalf

VIP Member

21

Years of Service

User Offline

Joined: 3rd Jan 2005

Playing: Malevolence:Sword of Ahkranox, Skyrim, Civ6.

Posted: 21st Dec 2011 02:05

Link

Looks OK to me. I can't see what I did wrong - unless it's something to do with the alpha component. That rings a bell for some reason. I'll check it carefully when I'm awake tomorrow and report back.

Back to top

Profile PM Email

TheComet

18

Years of Service

User Offline

Joined: 18th Oct 2007

Location: I`m under ur bridge eating ur goatz.

Posted: 21st Dec 2011 14:41

Link

Alright I await your response

ZIIIIIM

Back to top

Profile PM Email Website

Green Gandalf

VIP Member

21

Years of Service

User Offline

Joined: 3rd Jan 2005

Playing: Malevolence:Sword of Ahkranox, Skyrim, Civ6.

Posted: 23rd Dec 2011 19:48

Link

My apologies.

I forgot about this in the run-up to Xmas and am now away from a decent computer till 27th Dec. This means I can't test anything I suggest or even test Evolved's original demo. I'm sure there's a simple fix once I have a proper look at his demo and code.

I vaguely recall a similar issue coming up some time back so it might be worth doing a search.

If you're lucky someone else might put you out of your misery.

Hope you enjoy the pony steaks while you wait.

Back to top

Profile PM Email

MrValentine

AGK Backer

15

Years of Service

User Offline

Joined: 5th Dec 2010

Playing: FFVII

Posted: 23rd Dec 2011 21:59

Link

Interesting... I stall keep posted here on this topic...

Back to top

Profile PM

Green Gandalf

VIP Member

21

Years of Service

User Offline

Joined: 3rd Jan 2005

Playing: Malevolence:Sword of Ahkranox, Skyrim, Civ6.

Posted: 29th Dec 2011 01:37 Edited at: 29th Dec 2011 01:37

Link

TheComet

Is this any better? You may want to change the "shade.bmp" image as well to make it match the edge colour.

Here's the revised shader:

//====================================================
// Toon shader
//====================================================
// By EVOLVED
// www.evolved-software.com
//====================================================

//--------------
// un-tweaks
//--------------
   matrix WorldVP:WorldViewProjection; 
   matrix World:World;   
   matrix ViewInv:ViewInverse;

//--------------
// tweaks
//--------------
   float3 LightPosition={0.0f,50.0f,-250.0f};  
   float EdgeDis=0.4f;
   float4 EdgeColour={1.0, 0.0, 0.0, 1.0}; // full red, order is RGBA

//--------------
// Textures
//--------------
   texture BaseTX <string Name="";>;	
   sampler2D Base = sampler_state 
    {
 	texture = <BaseTX>;
    };
   texture ShadeTX <string Name="";>;	
   sampler2D Shade = sampler_state 
      {
 	texture = <ShadeTX>;
	MagFilter=None;
	MinFilter=None;
	MipFilter=None;	
      };

//--------------
// structs
//--------------
   struct input
      {
    	float4 Pos : POSITION ; 
    	float2 UV : TEXCOORD ; 
 	float3 Normal : NORMAL;  
      };
   struct output
      {
    	float4 OPos : POSITION ; 
    	float2 uv : TEXCOORD0 ; 
     	float2 Shade : TEXCOORD1 ;
      };
   struct output_Edge
      {
    	float4 OPos : POSITION ;
     	float4 Edge : COLOR0 ; 
      };

//--------------
// vertex shader
//--------------
   output VS(input IN) 
     {
    	output OUT;
    	OUT.OPos=mul(IN.Pos,WorldVP);
	OUT.uv=IN.UV;
	float3 WNor=mul(IN.Normal,World);  
	       WNor=normalize(WNor);
	float3 WPos=mul(IN.Pos,World); 
  	float3 LightVec=LightPosition-WPos;
   	float3 LightDir=mul(LightVec,WNor)/2;
	OUT.Shade.xy=saturate(LightDir/length(LightVec));
  	return OUT;
     }
   output_Edge VS_Edge(input IN) 
     {
    	output_Edge OUT;
	float4 Vpos;
	Vpos.xyz=IN.Pos+(IN.Normal*EdgeDis);Vpos.w=IN.Pos.w;
    	OUT.OPos=mul(Vpos,WorldVP);
	OUT.Edge.xyzw=EdgeColour;
  	return OUT;
     }

//--------------
// techniques 
//--------------
    technique Toon
      {
 	pass p0
      {		
 	vertexShader = compile vs_1_1 VS(); 
        Sampler[0] = <Base>;
  	ColorOp[0]   = MODULATE;
        ColorArg1[0] = TEXTURE;
        Sampler[1] = <Shade>;
  	ColorOp[1]   = MODULATE;
        ColorArg1[1] = TEXTURE;
      }
  	pass p1
      {		
 	vertexShader = compile vs_1_1 VS_Edge(); 
        CullMode = Cw;
//        Sampler[0] = null;
//        Sampler[1] = null;
        ColorOp[0] = disable;
        ColorOp[1] = disable;
      }
      }

+ Code Snippet

//====================================================
// Toon shader
//====================================================
// By EVOLVED
// www.evolved-software.com
//====================================================

//--------------
// un-tweaks
//--------------
   matrix WorldVP:WorldViewProjection; 
   matrix World:World;   
   matrix ViewInv:ViewInverse; 

//--------------
// tweaks
//--------------
   float3 LightPosition={0.0f,50.0f,-250.0f};  
   float EdgeDis=0.4f;
   float4 EdgeColour={1.0, 0.0, 0.0, 1.0}; // full red, order is RGBA

//--------------
// Textures
//--------------
   texture BaseTX <string Name="";>;	
   sampler2D Base = sampler_state 
    {
 	texture = <BaseTX>;
    };
   texture ShadeTX <string Name="";>;	
   sampler2D Shade = sampler_state 
      {
 	texture = <ShadeTX>;
	MagFilter=None;
	MinFilter=None;
	MipFilter=None;	
      };

//--------------
// structs
//--------------
   struct input
      {
    	float4 Pos : POSITION ; 
    	float2 UV : TEXCOORD ; 
 	float3 Normal : NORMAL;  
      };
   struct output
      {
    	float4 OPos : POSITION ; 
    	float2 uv : TEXCOORD0 ; 
     	float2 Shade : TEXCOORD1 ;
      };
   struct output_Edge
      {
    	float4 OPos : POSITION ;
     	float4 Edge : COLOR0 ; 
      };

//--------------
// vertex shader
//--------------
   output VS(input IN) 
     {
    	output OUT;
    	OUT.OPos=mul(IN.Pos,WorldVP);
	OUT.uv=IN.UV;
	float3 WNor=mul(IN.Normal,World);  
	       WNor=normalize(WNor);
	float3 WPos=mul(IN.Pos,World); 
  	float3 LightVec=LightPosition-WPos;
   	float3 LightDir=mul(LightVec,WNor)/2;
	OUT.Shade.xy=saturate(LightDir/length(LightVec));
  	return OUT;
     }
   output_Edge VS_Edge(input IN) 
     {
    	output_Edge OUT;
	float4 Vpos;
	Vpos.xyz=IN.Pos+(IN.Normal*EdgeDis);Vpos.w=IN.Pos.w;
    	OUT.OPos=mul(Vpos,WorldVP);
	OUT.Edge.xyzw=EdgeColour;
  	return OUT;
     }

//--------------
// techniques 
//--------------
    technique Toon
      {
 	pass p0
      {		
 	vertexShader = compile vs_1_1 VS(); 
        Sampler[0] = <Base>;
  	ColorOp[0]   = MODULATE;
        ColorArg1[0] = TEXTURE;
        Sampler[1] = <Shade>;
  	ColorOp[1]   = MODULATE;
        ColorArg1[1] = TEXTURE;
      }
  	pass p1
      {		
 	vertexShader = compile vs_1_1 VS_Edge(); 
        CullMode = Cw;
//        Sampler[0] = null;
//        Sampler[1] = null;
        ColorOp[0] = disable;
        ColorOp[1] = disable;
      }
      }

Here's a screenshot using the default red edge colour used in the shader.

Back to top

Profile PM Email

MrValentine

AGK Backer

15

Years of Service

User Offline

Joined: 5th Dec 2010

Playing: FFVII

Posted: 29th Dec 2011 01:44

Link

Like WOAH...

Back to top

Profile PM

Sorry your browser is not supported!

DarkBASIC Professional Discussion / Changing outline colour of toon shader?