Skip to main content

History: Enhanced CelShading

Preview of version: 5

UNDER CONSTRUCTION

Description

This is an enhanced version of Ogre's CelShading CG program and material. It adds the support of decal textures, dynamic ambient lighting, specular texture mapping, edge coloring, and light attenuation. Keep in mind that when using this shader, it will take more resources than the basic example that comes with Ogre.

The material and cg program below may look confusing, but do not worry, you do not have to understand the works completely in order to include it in your own materials.

CelShading.cg

Copy to clipboard
// //Basic lighting functions // void lighting_vp(float4 position : POSITION, float3 normal : NORMAL, // outputs out float4 oPosition : POSITION, out float diffuse : TEXCOORD0, // parameters uniform float3 lightPosition, // object space uniform float4 lightAttenuation, uniform float4x4 worldViewProj) { // calculate output position oPosition = mul(worldViewProj, position); // calculate light vector float3 N = normalize(normal); float3 L = normalize(lightPosition - position.xyz); //Calculate distance from light to object. float lightDist = length( lightPosition - position.xyz ); float luminosity = 0; if( lightAttenuation.x > lightDist) luminosity = 1.0 / ( lightAttenuation.y + lightAttenuation.z*lightDist + lightAttenuation.w*(lightDist*lightDist) ); // Calculate diffuse component diffuse = max(dot(N, L) , 0)*luminosity; } void lighting_fp(float diffuseIn : TEXCOORD0, out float4 colour : COLOR, uniform sampler1D diffuseRamp : register(s0)) { // Step functions from textures diffuseIn = tex1D(diffuseRamp, diffuseIn).x; colour = diffuseIn; } // //Specular // void specular_vp(float4 position : POSITION, float3 normal : NORMAL, // outputs out float4 oPosition : POSITION, out float specular : TEXCOORD0, float2 texCoord : TEXCOORD0, out float2 uv : TEXCOORD1, // parameters uniform float3 lightPosition, // object space uniform float4 lightAttenuation, uniform float3 eyePosition, // object space uniform float4 shininess, uniform float4x4 worldViewProj) { // calculate output position oPosition = mul(worldViewProj, position); // calculate light vector float3 N = normalize(normal); float3 L = normalize(lightPosition - position.xyz); // Calculate specular component float3 E = normalize(eyePosition - position.xyz); float3 H = normalize(L + E); //Calculate distance from light to object. float lightDist = length( lightPosition - position.xyz ); float luminosity = 0; if( lightAttenuation.x > lightDist) luminosity = 1.0 / ( lightAttenuation.y + lightAttenuation.z*lightDist + lightAttenuation.w*(lightDist*lightDist) ); specular = pow(max(dot(N, H), 0), shininess)*luminosity; uv = texCoord; } void specular_fp(float specularIn : TEXCOORD0, float2 uv : TEXCOORD1, out float4 colour : COLOR, uniform float4 specular, uniform sampler1D specularRamp : register(s0)) { // Step functions from textures specularIn = tex1D(specularRamp, specularIn).x; colour = specular * specularIn; } // //Specular with specular mapping image // void specularTex_fp(float specularIn : TEXCOORD0, float2 uv : TEXCOORD1, out float4 colour : COLOR, uniform float4 specular, uniform sampler1D specularRamp : register(s0), uniform sampler2D specMap : register(s1)) { // Step functions from textures specularIn = tex1D(specularRamp, specularIn).x; float4 specValue = tex2D(specMap, uv); colour = specular * specValue * specularIn; } // //Basic ambent lighting // void ambientOneTexture_vp(float4 position : POSITION, float2 uv : TEXCOORD0, out float4 oPosition : POSITION, out float2 oUv : TEXCOORD0, out float4 colour : COLOR, uniform float4x4 worldViewProj, uniform float4 ambient, uniform float4 diffuse) { oPosition = mul(worldViewProj, position); oUv = uv; colour = diffuse*ambient; } // //Edge shading // void celEdge_vp(float4 position : POSITION, float2 uv : TEXCOORD0, float3 normal : NORMAL, out float4 oPosition : POSITION, out float edge : TEXCOORD0, uniform float4x4 worldViewProj, uniform float3 eyePosition ) { oPosition = mul(worldViewProj, position); float3 N = normalize(normal); float3 E = normalize(eyePosition - position.xyz); edge = max(dot(N, E), 0); } void celEdge_fp(float edge : TEXCOORD0, out float4 colour : COLOR, uniform float4 edgeColor, uniform sampler1D edgeRamp : register(s0)) { edge = tex1D(edgeRamp, edge).x; colour = (1-edge)*edgeColor; }

CelShading.material

Copy to clipboard
// A really basic ambient pass program, support for one texture coodinate set vertex_program AmbientOneTexture cg { source CelShading.cg entry_point ambientOneTexture_vp profiles vs_1_1 arbvp1 default_params { param_named_auto worldViewProj worldviewproj_matrix param_named_auto ambient ambient_light_colour param_named diffuse float4 1 1 1 1 } } vertex_program CelEdgeVP cg { source CelShading.cg entry_point celEdge_vp profiles vs_1_1 arbvp1 default_params { param_named_auto worldViewProj worldviewproj_matrix param_named_auto eyePosition camera_position_object_space } } fragment_program CelEdgeFP cg { source CelShading.cg entry_point celEdge_fp profiles ps_1_1 arbfp1 fp20 default_params { param_named edgeColor float4 0 0 0 1 } } vertex_program CelShadingVP cg { source CelShading.cg entry_point lighting_vp profiles vs_1_1 arbvp1 default_params { param_named_auto lightPosition light_position_object_space 0 param_named_auto lightAttenuation light_attenuation 0 param_named_auto worldViewProj worldviewproj_matrix } } fragment_program CelShadingFP cg { source CelShading.cg entry_point lighting_fp profiles ps_1_1 arbfp1 fp20 } vertex_program CelSpecularVP cg { source CelShading.cg entry_point specular_vp profiles vs_1_1 arbvp1 default_params { param_named_auto lightPosition light_position_object_space 0 param_named_auto lightAttenuation light_attenuation 0 param_named_auto eyePosition camera_position_object_space param_named_auto worldViewProj worldviewproj_matrix param_named shininess float 10 } } fragment_program CelSpecularFP cg { source CelShading.cg entry_point specular_fp profiles ps_1_1 arbfp1 fp20 } fragment_program CelSpecularTexFP cg { source CelShading.cg entry_point specularTex_fp profiles ps_1_1 arbfp1 fp20 } material CelShading { technique { set $diffuse "1 1 1 1" set $specular "1 1 1 1" set $edge "0 0 0 1" set $shininess "25" // Base ambient pass pass ambient { // base colours, not needed for rendering, but as information // to lighting pass categorisation routine ambient 1 1 1 diffuse 0 0 0 specular 0 0 0 0 // Really basic vertex program // NB we don't use fixed function here because GL does not like // mixing fixed function and vertex programs, depth fighting can // be an issue vertex_program_ref AmbientOneTexture { } } pass perlight { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelShadingVP { } fragment_program_ref CelShadingFP { } texture_unit { texture cel_shading_diffuse.png 1d tex_address_mode clamp filtering none colour_op replace } } pass decal { lighting off vertex_program_ref AmbientOneTexture { param_named_auto worldViewProj worldviewproj_matrix param_named ambient float4 1 1 1 1 param_named diffuse float4 $diffuse } scene_blend dest_colour zero texture_unit decalmap { } } pass specular { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelSpecularVP { // map shininess from custom renderable param 1 param_named shininess float $shininess } fragment_program_ref CelSpecularFP { // map specular from custom renderable param 2 param_named specular float4 $specular } texture_unit { texture cel_shading_specular.png 1d tex_address_mode clamp filtering none colour_op replace } } pass edge { scene_blend alpha_blend // base colours, not needed for rendering, but as information // to lighting pass categorisation routine ambient 1 1 1 diffuse 0 0 0 specular 0 0 0 0 // Really basic vertex program // NB we don't use fixed function here because GL does not like // mixing fixed function and vertex programs, depth fighting can // be an issue vertex_program_ref CelEdgeVP { } fragment_program_ref CelEdgeFP { param_named edgeColor float4 $edge } texture_unit { texture cel_shading_edge.png 1d tex_address_mode clamp filtering none } } } } material CelShadingSpecular { technique { set $diffuse "1 1 1 1" set $specular "1 1 1 1" set $edge "0 0 0 1" set $shininess "25" // Base ambient pass pass ambient { // base colours, not needed for rendering, but as information // to lighting pass categorisation routine ambient 1 1 1 diffuse 0 0 0 specular 0 0 0 0 // Really basic vertex program // NB we don't use fixed function here because GL does not like // mixing fixed function and vertex programs, depth fighting can // be an issue vertex_program_ref AmbientOneTexture { } } pass perlight { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelShadingVP { } fragment_program_ref CelShadingFP { } texture_unit { texture cel_shading_diffuse.png 1d tex_address_mode clamp filtering none colour_op replace } } pass decal { lighting off vertex_program_ref AmbientOneTexture { param_named_auto worldViewProj worldviewproj_matrix param_named ambient float4 1 1 1 1 param_named diffuse float4 $diffuse } scene_blend dest_colour zero texture_unit decalmap { } } pass specular { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelSpecularVP { // map shininess from custom renderable param 1 param_named shininess float $shininess } fragment_program_ref CelSpecularTexFP { // map specular from custom renderable param 2 param_named specular float4 $specular } texture_unit { texture cel_shading_specular.png 1d tex_address_mode clamp filtering none colour_op replace } texture_unit specMap { texture white.png } } pass edge { scene_blend alpha_blend // base colours, not needed for rendering, but as information // to lighting pass categorisation routine ambient 1 1 1 diffuse 0 0 0 specular 0 0 0 0 // Really basic vertex program // NB we don't use fixed function here because GL does not like // mixing fixed function and vertex programs, depth fighting can // be an issue vertex_program_ref CelEdgeVP { } fragment_program_ref CelEdgeFP { param_named edgeColor float4 $edge } texture_unit { texture cel_shading_edge.png 1d tex_address_mode clamp filtering none } } } }

Usage

You need to have three textures in order for the material to work, you can get them from Ogre's media/materials/textures directory: cel_shading_edge.png, cel_shading_diffuse.png, and cel_shading_specular.png. You will have to modify cel_shading_diffuse.png and change the left edge of the image to black instead of grey.

You can define two variables very much like in regular materials: $diffuse and $specular. There are two more variables: $edge and $shininess, edge is the color of the outline over the mesh. For texturing, you would define the pass: decal, then use the texture_unit: decalmap, and insert your texture.

  • $diffuse -This is a 4 vector variable that controls the color of the mesh or submesh. The first three values are red, green, and blue in that order. The last value is alpha which is the transparency of the mesh/submesh where "0" is completely invisible and "1" is completely visible.
  • $specular -This is a 4 vector variable like diffuse only it controls the color of the "shine" on the mesh.
  • $edge -This 4 vector variable controls the color of the outline on the mesh.
  • $shininess - This single float variable controls the shine intensity of the mesh.


This is an example material overriding the OgreHead Mesh's materials with the CelShading materials:

Copy to clipboard
import * from "CelShading.material" material Ogre/Skin : CelShadingSpecular { technique { set $specular "0.3 0.3 0.3 1" pass decal { texture_unit decalmap { texture GreenSkin.jpg } } pass specular { texture_unit specMap { texture GreenSkin.jpg } } } } material Ogre/Eyes : CelShading { technique { set $diffuse "1 0.7 0 1" set $specular "1 0.8 0 1" set $edge "0 0 0 0" } } material Ogre/Tusks : CelShading { technique { set $diffuse "0.7 0.7 0.6 1" set $specular "0 0 0 1" set $edge "0.4 0.4 0.3 1" pass decal { texture_unit decalmap { texture tusk.jpg } } } } material Ogre/Earring : CelShading { technique { set $diffuse "0.7 0.7 0 1" set $specular "1 1 1 1" set $edge "0.6 0.6 0 1" } }

Experimental Edge Outlining


I did a little experimenting with different ways that you can add an edge outline around a mesh. Many are not satisfied with the current outlining because it is cheap and only looks good on smooth models. This is also a cheap workaround for the "official" way to add cartoon outlines to a mesh but I found that it looks better and works with less smooth models. A link to the forum thread can be found here: http://www.ogre3d.org/forums/viewtopic.php?p=422160#p422160

Two new tweak-able variables have been added to the material script:

  • $modscale -This variable must be tweaked to fit your model. Usually a larger number for a larger/complex mesh. Currently the best value I found for the Ogre Head mesh is "4.0"
  • $edgescale -This is the scale of the edge around the mesh, "1.0" is the default scale and "2.0" makes it twice as large as it is.


(Note: You must keep the file CelShading.cg in your project from the code above for this to work)

If you were to use this experimental edge detection, add a new file to your project and call it "VertEdge.cg" and add this code into it:
Copy to clipboard
//******************************************************************* void EdgeVP(float4 position : POSITION, float4 normal : NORMAL0, out float4 oPos : POSITION, out float4 oColor : COLOR, uniform float4 edgeColor, uniform float4 eyePosition, uniform float scale, uniform float edgeScale, uniform float4x4 worldViewProj) { float4 E = normalize(eyePosition); position = mul(worldViewProj, position - scale*E); normal.w = 0; normal = normalize(mul(worldViewProj, normal)); //position -= float4(normal.xy*2, normal.zw); position += ((scale/8.0f)+1.0) * edgeScale * float4(normal.xy, 0, 0); oPos=position; oColor=edgeColor; }

Next, replace the contents of CelShading.material with this:
Copy to clipboard
vertex_program AmbientOneTexture cg { source CelShading.cg entry_point ambientOneTexture_vp profiles vs_1_1 arbvp1 default_params { param_named_auto worldViewProj worldviewproj_matrix param_named_auto ambient ambient_light_colour param_named diffuse float4 1 1 1 1 } } vertex_program CelEdgeVP cg { source CelShading.cg entry_point celEdge_vp profiles vs_1_1 arbvp1 default_params { param_named_auto worldViewProj worldviewproj_matrix param_named_auto eyePosition camera_position_object_space } } fragment_program CelEdgeFP cg { source CelShading.cg entry_point celEdge_fp profiles ps_1_1 arbfp1 fp20 default_params { param_named edgeColor float4 0 0 0 1 } } vertex_program VertEdge cg { source VertEdge.cg entry_point EdgeVP profiles vs_1_1 arbvp1 default_params { param_named_auto worldViewProj worldviewproj_matrix param_named_auto eyePosition camera_position_object_space param_named scale float 2 param_named edgeScale float 1 } } vertex_program CelShadingVP cg { source CelShading.cg entry_point lighting_vp profiles vs_1_1 arbvp1 default_params { param_named_auto lightPosition light_position_object_space 0 param_named_auto lightAttenuation light_attenuation 0 param_named_auto worldViewProj worldviewproj_matrix } } fragment_program CelShadingFP cg { source CelShading.cg entry_point lighting_fp profiles ps_1_1 arbfp1 fp20 } vertex_program CelSpecularVP cg { source CelShading.cg entry_point specular_vp profiles vs_1_1 arbvp1 default_params { param_named_auto lightPosition light_position_object_space 0 param_named_auto lightAttenuation light_attenuation 0 param_named_auto eyePosition camera_position_object_space param_named_auto worldViewProj worldviewproj_matrix param_named shininess float 10 } } fragment_program CelSpecularFP cg { source CelShading.cg entry_point specular_fp profiles ps_1_1 arbfp1 fp20 } fragment_program CelSpecularTexFP cg { source CelShading.cg entry_point specularTex_fp profiles ps_1_1 arbfp1 fp20 } material CelShading { technique { set $diffuse "1 1 1 1" set $specular "1 1 1 1" set $edge "0 0 0 1" set $shininess "25" set $modscale "2" set $edgescale "1" pass edge { vertex_program_ref VertEdge { param_named edgeColor float4 $edge param_named scale float $modscale param_named edgeScale float $edgescale } } // Base ambient pass pass ambient { // base colours, not needed for rendering, but as information // to lighting pass categorisation routine ambient 1 1 1 diffuse 0 0 0 specular 0 0 0 0 // Really basic vertex program // NB we don't use fixed function here because GL does not like // mixing fixed function and vertex programs, depth fighting can // be an issue vertex_program_ref AmbientOneTexture { } } pass perlight { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelShadingVP { } fragment_program_ref CelShadingFP { } texture_unit { texture cel_shading_diffuse.png 1d tex_address_mode clamp filtering trilinear colour_op replace } } pass decal { lighting off vertex_program_ref AmbientOneTexture { param_named_auto worldViewProj worldviewproj_matrix param_named ambient float4 1 1 1 1 param_named diffuse float4 $diffuse } scene_blend dest_colour zero texture_unit decalmap { } } pass specular { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelSpecularVP { // map shininess from custom renderable param 1 param_named shininess float $shininess } fragment_program_ref CelSpecularFP { // map specular from custom renderable param 2 param_named specular float4 $specular } texture_unit { texture cel_shading_specular.png 1d tex_address_mode clamp filtering trilinear colour_op replace } } } } material CelShadingSpecular { technique { set $diffuse "1 1 1 1" set $specular "1 1 1 1" set $edge "0 0 0 1" set $shininess "25" set $modscale "2" set $edgescale "1" pass edge { vertex_program_ref VertEdge { param_named edgeColor float4 $edge param_named scale float $modscale param_named edgeScale float $edgescale } } // Base ambient pass pass ambient { // base colours, not needed for rendering, but as information // to lighting pass categorisation routine ambient 1 1 1 diffuse 0 0 0 specular 0 0 0 0 // Really basic vertex program // NB we don't use fixed function here because GL does not like // mixing fixed function and vertex programs, depth fighting can // be an issue vertex_program_ref AmbientOneTexture { } } pass perlight { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelShadingVP { } fragment_program_ref CelShadingFP { } texture_unit { texture cel_shading_diffuse.png 1d tex_address_mode clamp filtering trilinear colour_op replace } } pass decal { lighting off vertex_program_ref AmbientOneTexture { param_named_auto worldViewProj worldviewproj_matrix param_named ambient float4 1 1 1 1 param_named diffuse float4 $diffuse } scene_blend dest_colour zero texture_unit decalmap { } } pass specular { ambient 0 0 0 iteration once_per_light scene_blend add vertex_program_ref CelSpecularVP { // map shininess from custom renderable param 1 param_named shininess float $shininess } fragment_program_ref CelSpecularTexFP { // map specular from custom renderable param 2 param_named specular float4 $specular } texture_unit { texture cel_shading_specular.png 1d tex_address_mode clamp filtering trilinear colour_op replace } texture_unit specMap { texture white.png } } } }

Below is an example code that replaces the Ogre Head's materials with these cel shaded ones:
Copy to clipboard
import * from "CelShading.material" //For the Ogre/Skin material we are inheriting the material: CelShadingSpecular, // which is a version of the material that supports specular mapping with a texture. material Ogre/Skin : CelShadingSpecular { technique { set $specular "0.3 0.3 0.3 1" set $modscale "4.0" pass decal { texture_unit decalmap { texture GreenSkin.jpg } } pass specular { texture_unit specMap { texture GreenSkin.jpg } } } } //Specular texture mapping turned off for the rest of the materials material Ogre/Eyes : CelShading { technique { set $diffuse "1 0.7 0 1" set $specular "1 0.8 0 1" set $edge "0 0 0 0" //Note: we do not add the $modscale variable here because the default value fits this material } } material Ogre/Tusks : CelShading { technique { set $diffuse "0.7 0.7 0.6 1" set $specular "0 0 0 1" set $edge "0.4 0.4 0.3 1" pass decal { texture_unit decalmap { texture tusk.jpg } } } } material Ogre/Earring : CelShading { technique { set $diffuse "0.7 0.7 0 1" set $specular "1 1 1 1" set $edge "0.6 0.6 0 1" } }

The Code

History

Information Version
Wed 14 of May, 2025 15:49 GMT-0000 admin attachment conversion 14
Fri 27 of Apr, 2012 00:09 GMT-0000 StrakeFengala 13
Sat 21 of Apr, 2012 01:40 GMT-0000 StrakeFengala 12
Sat 21 of Apr, 2012 01:39 GMT-0000 StrakeFengala Updated ATTENTION text 11
Fri 20 of Apr, 2012 22:36 GMT-0000 StrakeFengala 10
Fri 20 of Apr, 2012 20:23 GMT-0000 StrakeFengala 9
Fri 20 of Apr, 2012 16:53 GMT-0000 StrakeFengala 8
Fri 20 of Apr, 2012 16:48 GMT-0000 StrakeFengala 7
Fri 20 of Apr, 2012 16:46 GMT-0000 StrakeFengala Major revamp 6
Fri 20 of Apr, 2012 15:55 GMT-0000 StrakeFengala 5
Fri 20 of Apr, 2012 15:54 GMT-0000 StrakeFengala 4
Sun 31 of Jul, 2011 22:21 GMT-0000 jacmoe 3
Sun 13 of Mar, 2011 03:22 GMT-0000 StrakeFengala 2
Sun 13 of Mar, 2011 03:19 GMT-0000 StrakeFengala 1