How to use Unity3d shader to mask another two-pass shader? - unity3d

Question

I am developing a game where I want to visualize user path - showing also where track overlaps (with some darker color than normal "not overlapped" track). Long time ago I have found this answer where it was advised to use stencil to create two-pass shader. That day it was very helpful.
But lately I have met an additional requirement: I want to expose the path (part of mesh) which is inside some irregular area. For the rest of the path there are two options based on user choice in the runtime:
should be marked with any other different color (e.g. darker),
should be not visible at all.
I have found that answer as a suggestion to use stencil in another shader which will be put to some mesh to create a "mask". I made some adjustments and I ended up with those shaders:
Mask shader:
Shader "Custom/AreaValidator"
{
Properties
{
_Color ("Color", Color) = (1,1,1,1)
_MainTex ("Albedo (RGB)", 2D) = "white" {}
_Glossiness ("Smoothness", Range(0,1)) = 0.5
_Metallic ("Metallic", Range(0,1)) = 0.0
}
SubShader
{
Tags { "RenderType"="Opaque" "Queue"="Geometry-1" }
ColorMask 0
ZWrite off
Stencil{
Ref 3
Comp always
Pass replace
}
CGPROGRAM
// Physically based Standard lighting model, and enable shadows on all light types
#pragma surface surf Standard fullforwardshadows
// Use shader model 3.0 target, to get nicer looking lighting
#pragma target 3.0
sampler2D _MainTex;
struct Input
{
float2 uv_MainTex;
};
half _Glossiness;
half _Metallic;
fixed4 _Color;
// Add instancing support for this shader. You need to check 'Enable Instancing' on materials that use the shader.
// See https://docs.unity3d.com/Manual/GPUInstancing.html for more information about instancing.
// #pragma instancing_options assumeuniformscaling
UNITY_INSTANCING_BUFFER_START(Props)
// put more per-instance properties here
UNITY_INSTANCING_BUFFER_END(Props)
void surf (Input IN, inout SurfaceOutputStandard o)
{
// Albedo comes from a texture tinted by color
fixed4 c = tex2D (_MainTex, IN.uv_MainTex) * _Color;
o.Albedo = c.rgb;
// Metallic and smoothness come from slider variables
o.Metallic = _Metallic;
o.Smoothness = _Glossiness;
o.Alpha = c.a;
}
ENDCG
}
FallBack "Diffuse"
}
And path shader:
Shader "Unlit/Path"
{
Properties
{
_MainTex ("Texture", 2D) = "white" {}
}
SubShader
{
Tags { "RenderType"="Opaque" }
LOD 100
Pass
{
Stencil {
Ref 0
Comp Equal
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct appdata
{
float4 vertex : POSITION;
};
struct v2f
{
float4 vertex : SV_POSITION;
};
v2f vert (appdata v)
{
v2f o;
o.vertex = UnityObjectToClipPos(v.vertex);
return o;
}
fixed4 frag (v2f i) : SV_Target
{
fixed4 col = fixed4(0.545, 0.780, 0.341, 0.8);
return col;
}
ENDCG
}
Pass
{
Stencil {
Ref 3
Comp Equal
Pass IncrSat
Fail IncrSat
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct appdata
{
float4 vertex : POSITION;
};
struct v2f
{
float4 vertex : SV_POSITION;
};
v2f vert (appdata v)
{
v2f o;
o.vertex = UnityObjectToClipPos(v.vertex);
return o;
}
fixed4 frag (v2f i) : SV_Target
{
fixed4 col = fixed4(0.710, 0.859, 0.580, 1);
return col;
}
ENDCG
}
Pass
{
Stencil {
Ref 4
Comp Less
}
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct appdata
{
float4 vertex : POSITION;
};
struct v2f
{
float4 vertex : SV_POSITION;
};
v2f vert (appdata v)
{
v2f o;
o.vertex = UnityObjectToClipPos(v.vertex);
return o;
}
fixed4 frag (v2f i) : SV_Target
{
fixed4 col = fixed4(0.545, 0.780, 0.341, 0.8);
return col;
}
ENDCG
}
}
}
As you can see in mask shader I have:
Stencil{
Ref 3
Comp always
Pass replace
}
but in path shader I have 3 passes: for "outside validated area", "inside validated area - without overlaps", "inside validated area - overlapping".
1.
Stencil {
Ref 0
Comp Equal
}
2.
Stencil {
Ref 3
Comp Equal
Pass IncrSat
Fail IncrSat
}
3.
Stencil {
Ref 4
Comp Less
}
I do not fully understand how shaders are rendered - I am showing the solution which was the closest to my goal. In general it "works" in some simple cases:
here - all tracks visible (outside valid area color is darker like
during overlapping),
here - when I remove from shader code full section responsible for first pass the track outside area is removed - that is good,
but there are 3 major problems which I cannot solve.
First problem - "glitching/flickering" when 2 different meshes are overlapping:
Another problem- because stencil 2) is incrementing buffer in both pass/fail situations when user goes around few times then even outside "valid" area track will be rendered.
Third problem is that I do not know how switch between "visible outside track" and "not visible outside track" rendering options during runtime.
Summing up:
I would like to have a mesh:
which is rendered differently when it overlaps,
can be render only inside certain "masking" area,
user is able to show/hide mesh outside masking area dynamically, visible mesh outside has other color,
but I do not know how to achieve that without glitches and some weird artifacts.