//======= Copyright (c) Stereolabs Corporation, All rights reserved. ===============
using UnityEngine;
using System.IO;
#if ZED_HDRP || ZED_URP
using UnityEngine.Rendering;
#endif
///
/// When attached to an object that also has a ZEDRenderingPlane, removes all real-world pixels
/// of a specified color. Useful for third-person mixed reality setups.
/// Also requires the Frame object of the ZEDRenderingPlane component to have its MeshRenderer material set to Mat_ZED_GreenScreen.
/// The simplest way to use the plugin's GreenScreen feature is to use the ZED_GreenScreen prefab in the GreenScreen sample folder.
/// For detailed information on using the ZED for greenscreen effects, see https://docs.stereolabs.com/mixed-reality/unity/green-screen-vr/.
///
//[RequireComponent(typeof(ZEDManager))]
public class GreenScreenManager : MonoBehaviour
{
///
/// The plane used for rendering. Equal to the canvas value of ZEDRenderingPlane.
///
private GameObject screen = null;
///
/// The Camera that this script is attached to. Only needed in SRP for a callback, since OnPreRender won't work.
///
private Camera cam;
///
/// The screen manager script. Automatically assigned in OnEnable().
///
public ZEDManager cameraManager = null;
///
/// The screen manager script. Automatically assigned in OnEnable().
///
public ZEDRenderingPlane screenManager = null;
///
/// Set to true when there is data available from a configuration file that needs to be loaded.
/// It will then be loaded the next call to Update().
///
private bool toUpdateConfig = false;
///
/// Holds chroma key settings together in a single serializable object.
///
[System.Serializable]
public struct ChromaKey
{
public Color color;
public float smoothness;
public float range;
}
///
/// Holds chroma key data together in a single serializable object.
///
[System.Serializable]
public struct ChromaKeyData
{
public ChromaKey chromaKeys;
public int erosion;
public int numberBlurIterations;
public float blurPower;
public float whiteClip;
public float blackClip;
public float spill;
}
///
/// Holds greenscreen and garbage matte configuration data together in a single serializable object.
///
[System.Serializable]
public struct GreenScreenData
{
public ChromaKeyData chromaKeyData;
public GarbageMatte.GarbageMatteData garbageMatteData;
}
///
/// Array of available chroma key settings
///
public ChromaKey keys;
///
/// Array of available key colors
///
[SerializeField]
public Color keyColors = new Color(0.0f, 1.0f, 0.0f, 1);
///
/// Causes pixels on the edge of the range to the color to fade out, instead of all pixels being 100% or 0% visible.
///
[SerializeField]
public float smoothness;
///
/// Governs how similar a pixel must be to the chosen color to get removed.
///
[SerializeField]
public float range;
///
/// subtracts the color value from the foreground image, making it appear "less green" for instance.
/// Useful because bright lighting can cause the color of a greenscreen to spill onto your actual subject.
///
[SerializeField]
public float spill = 0.2f;
///
/// Default green color for the chroma key.
///
private Color defaultColor = new Color(0.0f, 1.0f, 0.0f, 1);
///
/// Default Smoothness value.
///
private const float defaultSmoothness = 0.08f;
///
/// Default Range value.
///
private const float defaultRange = 0.42f;
///
/// Default Spill value.
///
private const float defaultSpill = 0.1f;
///
/// Default Erosion value.
///
private const int defaultErosion = 0;
///
/// Default White Clip value.
///
private const float defaultWhiteClip = 1.0f;
///
/// Default Black Clip value.
///
private const float defaultBlackClip = 0.0f;
///
/// Default sigma.
///
private const float defaultSigma = 0.1f;
///
/// Final rendering material, eg. the material on the ZEDRenderingPlane's canvas object.
///
public Material finalMat;
///
/// Green screen effect material.
///
private Material greenScreenMat;
///
/// Material used to apply preprocessing effects in OnPreRender.
///
private Material preprocessMat;
///
/// Alpha texture for blending.
///
private RenderTexture finalTexture;
///
/// Public accessor for the alpha texture used for blending.
///
public RenderTexture FinalTexture
{
get { return finalTexture; }
}
///
/// Available canals (views) for displaying the chroma key effect.
///
public enum CANAL
{
FOREGROUND,
BACKGROUND,
ALPHA,
KEY,
FINAL
};
///
/// The current view. Change to see different steps of the rendering stage, which can be helpful for tweaking other settings.
///
[HideInInspector]
[SerializeField]
public CANAL canal = CANAL.FINAL;
///
/// Carves off pixels from the edges between the foreground and background.
///
[HideInInspector]
[SerializeField]
public int erosion = 0;
///
/// Causes pixels with alpha values above its setting to be set to 100% alpha, useful for reducing noise resulting from the smoothness setting.
///
[SerializeField]
public float whiteClip = 1.0f;
///
/// Causes pixels with alpha values below its setting to be set to 0% alpha, useful for reducing noise resulting from the smoothness setting.
///
[SerializeField]
public float blackClip = 0.0f;
///
/// The path to the .config file, where configurations would be loaded or saved when you press the relevant button.
///
[SerializeField]
public string pathFileConfig = "Assets/Config_greenscreen.json";
///
/// Material used to apply blur effect in OnPreRender().
///
private Material blurMaterial;
///
/// Blur iteration number. A larger value increases the blur effect.
///
public int numberBlurIterations = 5;
///
/// Sigma value. A larger value increases the blur effect.
///
public float sigma_ = 0.1f;
///
/// Current sigma value.
///
private float currentSigma = -1;
///
/// Weights for blur effect.
///
private float[] weights_ = new float[5];
///
/// Offsets for blur effect.
///
private float[] offsets_ = { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f };
///
/// Material used to convert an RGBA texture into YUV, which makes it simpler to add certain effects.
///
public Material matYUV;
///
/// Reference to the currently active GarbageMatte.
/// We make an empty one regardless of the enableGarbageMatte setting so a reference to one is always available.
///
[SerializeField]
[HideInInspector]
public GarbageMatte garbageMatte = new GarbageMatte();
///
/// Whether we're using the garbage matte effect or not.
///
public bool enableGarbageMatte = false;
///
/// All the data for the garbage matte.
///
private GarbageMatte.GarbageMatteData garbageMatteData;
///
/// Sets all settings to the default values. Called on a new component, and when Reset is clicked.
///
public void SetDefaultValues()
{
keyColors = defaultColor;
smoothness = defaultSmoothness;
range = defaultRange;
spill = defaultSpill;
erosion = defaultErosion;
whiteClip = defaultWhiteClip;
blackClip = defaultBlackClip;
sigma_ = defaultSigma;
}
///
/// Cached property id for _MaskTex. use the maskTexID property instead.
///
private int? _masktexid;
///
/// Property id for _MaskTex, which is the RenderTexture property for the mask texture.
///
private int maskTexID
{
get
{
if (_masktexid == null) _masktexid = Shader.PropertyToID("_MaskTex");
return (int)_masktexid;
}
}
///
/// Cached property id for weights. use the weightsID property instead.
///
private int? _weightsid;
///
/// Property id for weights, which affects post-processing blurring.
///
private int weightsID
{
get
{
if (_masktexid == null) _masktexid = Shader.PropertyToID("weights");
return (int)_masktexid;
}
}
///
/// Cached property id for offset. use the offsetID property instead.
///
private int? _offsetid;
///
/// Property id for offset, which affects post-processing blurring.
///
private int offsetID
{
get
{
if (_offsetid == null) _offsetid = Shader.PropertyToID("offset");
return (int)_offsetid;
}
}
private void OnEnable()
{
//cameraManager = gameObject.GetComponent ();
cameraManager = gameObject.GetComponentInParent();
if (!cameraManager)
{
//cameraManager = ZEDManager.GetInstance(sl.ZED_CAMERA_ID.CAMERA_ID_01);
cameraManager = FindObjectOfType();
}
cameraManager.OnZEDReady += ZEDReady;
Shader.SetGlobalInt("ZEDGreenScreenActivated", 1);
screenManager = GetComponent();
}
private void OnDisable()
{
if (cameraManager)
{
cameraManager.OnZEDReady -= ZEDReady;
}
}
private void Awake()
{
//cameraManager = gameObject.GetComponent ();
Shader.SetGlobalInt("_ZEDStencilComp", 0);
if (screen == null)
{
screen = gameObject.transform.GetChild(0).gameObject;
finalMat = screen.GetComponent().material;
}
if (enableGarbageMatte)
{
garbageMatte = new GarbageMatte(cameraManager, finalMat, transform, garbageMatte);
}
#if ZED_HDRP || ZED_URP
cam = GetComponent();
if (!cam) Debug.LogError("GreenScreenManager is not attached to a Camera.");
RenderPipelineManager.beginCameraRendering += OnSRPPreRender;
#endif
#if !UNITY_EDITOR
Debug.Log("Load Chroma keys");
LoadGreenScreenData();
UpdateShader();
CreateFileWatcher("");
#endif
}
///
/// Holds whether we need to apply the DEPTH_ALPHA keyword to the ZED imagefirst pass material (Mat_ZED_Forward)
/// which causes it to process the greenscreen mask.
///
private bool textureOverlayInit = false;
private void Update()
{
if (screenManager != null && !textureOverlayInit) //Need to tell the shader to apply the mask.
{
if (screenManager.ManageKeywordForwardMat(true, "DEPTH_ALPHA"))
{
textureOverlayInit = true;
}
}
if (toUpdateConfig) //Need to load available greenscreen configuration data into the current, active data.
{
toUpdateConfig = false;
LoadGreenScreenData();
}
if (enableGarbageMatte) //Set up the garbage matte if needed.
{
if (garbageMatte != null && garbageMatte.IsInit)
{
garbageMatte.Update();
}
else
{
if (cameraManager != null)
garbageMatte = new GarbageMatte(cameraManager, finalMat, transform, garbageMatte);
}
}
}
///
/// Initialization logic that must be done after the ZED camera has finished initializing.
/// Added to the ZEDManager.OnZEDReady() callback in OnEnable().
///
private void ZEDReady()
{
//Set up textures and materials used for the final output.
if (cameraManager == null)
return;
//We set the material again in case it has changed.
//screen = gameObject.transform.GetChild(0).gameObject;
finalMat = screenManager.matRGB;
finalTexture = new RenderTexture(cameraManager.zedCamera.ImageWidth, cameraManager.zedCamera.ImageHeight, 0, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Linear);
if (screenManager.forwardMat) screenManager.forwardMat.SetTexture("ZEDMaskTexGreenScreen", finalTexture);
if (screenManager.deferredMat) screenManager.deferredMat.SetTexture("ZEDMaskTexGreenScreen", finalTexture);
finalMat.SetTexture("_MaskTex", finalTexture);
greenScreenMat = new Material(Resources.Load("Materials/Mat_ZED_Compute_GreenScreen") as Material);
blurMaterial = new Material(Resources.Load("Materials/PostProcessing/Mat_ZED_Blur") as Material);
matYUV = new Material(Resources.Load("Materials/Mat_ZED_YUV") as Material);
matYUV.SetInt("_isLinear", System.Convert.ToInt32(QualitySettings.activeColorSpace));
preprocessMat = new Material(Resources.Load("Materials/Mat_ZED_Preprocess") as Material);
preprocessMat.SetTexture("_CameraTex", screenManager.TextureEye);
ZEDPostProcessingTools.ComputeWeights(1, out weights_, out offsets_);
//Send the values to the current shader.
blurMaterial.SetFloatArray("weights2", weights_);
blurMaterial.SetFloatArray("offset2", offsets_);
greenScreenMat.SetTexture("_CameraTex", screenManager.TextureEye);
UpdateShader();
UpdateCanal();
if (System.IO.File.Exists("ZED_Settings.conf"))
{
cameraManager.zedCamera.LoadCameraSettings("ZED_Settings.conf");
cameraManager.zedCamera.SetCameraSettings();
}
}
///
/// Update the current canal (VIEW) to the setting in the editor.
/// The greenscreen shader uses #ifdef keywords to render differently depending on the active canal.
/// This class makes sure that exactly one such keyword is defined at one time.
///
public void UpdateCanal()
{
foreach (CANAL c in System.Enum.GetValues(typeof(CANAL))) //Clear all keywords
{
manageKeyWord(false, c.ToString());
}
if (screenManager != null)
{
//Set NO_DEPTH keyword as well if set to BACKGROUND.
if (canal == CANAL.BACKGROUND) screenManager.ManageKeywordForwardMat(true, "NO_DEPTH");
else screenManager.ManageKeywordForwardMat(false, "NO_DEPTH");
manageKeyWord(true, canal.ToString()); //Activate the keyword corresponding to the current Canal.
}
}
///
/// Enables or disables a shader keyword in the finalMat material.
///
///
///
void manageKeyWord(bool value, string name)
{
if (finalMat != null)
{
if (value)
{
finalMat.EnableKeyword(name);
}
else
{
finalMat.DisableKeyword(name);
}
}
}
#if UNITY_EDITOR
private void OnValidate() //When the Inspector is visible and it changes somehow.
{
UpdateShader();
}
#endif
private void OnApplicationQuit()
{
if (finalTexture != null && finalTexture.IsCreated()) finalTexture.Release();
}
public void Reset() //When the Unity editor Reset button is used.
{
cameraManager = gameObject.GetComponentInParent();
if (!cameraManager)
{
cameraManager = ZEDManager.GetInstance(sl.ZED_CAMERA_ID.CAMERA_ID_01);
}
if (cameraManager != null)
{
cameraManager.resolution = sl.RESOLUTION.HD1080;
Debug.Log("Resolution set to HD1080 for better result");
}
SetDefaultValues();
}
///
/// Helper function to convert colors in RGB color space to YUV format.
///
/// Color to be converted.
/// Whether to keep the values within the maximum.
///
Vector3 RGBtoYUV(Color rgb, bool clamped = true)
{
double Y = 0.182586f * rgb.r + 0.614231f * rgb.g + 0.062007f * rgb.b + 0.062745f; // Luma
double U = -0.100644f * rgb.r - 0.338572f * rgb.g + 0.439216f * rgb.b + 0.501961f; // Delta Blue
double V = 0.439216f * rgb.r - 0.398942f * rgb.g - 0.040274f * rgb.b + 0.501961f; // Delta Red
if (!clamped)
{
U = -0.100644 * rgb.r - 0.338572 * rgb.g + 0.439216 * rgb.b; // Delta Blue
V = 0.439216 * rgb.r - 0.398942 * rgb.g - 0.040274 * rgb.b; // Delta Red
}
return new Vector3((float)Y, (float)U, (float)V);
}
///
/// Update all the data to the greenscreen shader.
/// The weights and offsets will be set when sigma changes.
///
public void UpdateShader()
{
if (greenScreenMat != null)
{
greenScreenMat.SetVector("_keyColor", RGBtoYUV(keyColors));
greenScreenMat.SetFloat("_range", range);
preprocessMat.SetFloat("_erosion", erosion);
preprocessMat.SetFloat("_smoothness", smoothness);
preprocessMat.SetFloat("_whiteClip", whiteClip);
preprocessMat.SetFloat("_blackClip", blackClip);
preprocessMat.SetFloat("_spill", spill);
preprocessMat.SetColor("_keyColor", keyColors);
}
}
///
/// Load the data from a file and fills a structure.
///
/// Whether there's a valid file where pathFileConfig says there is.
private bool LoadData(out GreenScreenData gsData)
{
gsData = new GreenScreenData();
if (File.Exists(pathFileConfig))
{
string dataAsJson = File.ReadAllText(pathFileConfig);
gsData = JsonUtility.FromJson(dataAsJson);
return true;
}
return false;
}
///
/// Creates a new serializable ChromaKeyData file from the current settings.
///
public ChromaKeyData RegisterDataChromaKeys()
{
ChromaKeyData chromaKeyData = new ChromaKeyData();
chromaKeyData.chromaKeys = new ChromaKey();
chromaKeyData.erosion = erosion;
chromaKeyData.blurPower = sigma_;
chromaKeyData.numberBlurIterations = numberBlurIterations;
chromaKeyData.whiteClip = whiteClip;
chromaKeyData.spill = spill;
chromaKeyData.blackClip = blackClip;
chromaKeyData.chromaKeys.color = keyColors;
chromaKeyData.chromaKeys.smoothness = smoothness;
chromaKeyData.chromaKeys.range = range;
return chromaKeyData;
}
///
/// Saves the chroma keys used in a file (JSON format).
///
public void SaveData(ChromaKeyData chromaKeyData, GarbageMatte.GarbageMatteData garbageMatteData)
{
GreenScreenData gsData = new GreenScreenData();
gsData.chromaKeyData = chromaKeyData;
gsData.garbageMatteData = garbageMatteData;
string dataAsJson = JsonUtility.ToJson(gsData);
File.WriteAllText(pathFileConfig, dataAsJson);
}
///
/// Fills the current chroma keys with the data from a file.
///
public void LoadGreenScreenData(bool forcegarbagemate = false)
{
GreenScreenData gsData;
if (LoadData(out gsData))
{
ChromaKeyData chromaKeyData = gsData.chromaKeyData;
garbageMatteData = gsData.garbageMatteData;
erosion = chromaKeyData.erosion;
sigma_ = chromaKeyData.blurPower;
numberBlurIterations = chromaKeyData.numberBlurIterations;
whiteClip = chromaKeyData.whiteClip;
blackClip = chromaKeyData.blackClip;
spill = chromaKeyData.spill;
keyColors = chromaKeyData.chromaKeys.color;
smoothness = chromaKeyData.chromaKeys.smoothness;
range = chromaKeyData.chromaKeys.range;
}
UpdateShader();
if (forcegarbagemate && garbageMatte != null)
{
if (!garbageMatte.IsInit && cameraManager != null)
{
garbageMatte = new GarbageMatte(cameraManager, finalMat, transform, garbageMatte);
}
enableGarbageMatte = true;
garbageMatte.LoadData(gsData.garbageMatteData);
garbageMatte.ApplyGarbageMatte();
}
}
#if ZED_HDRP || ZED_URP
///
///
///
///
private void OnSRPPreRender(ScriptableRenderContext context, Camera renderingcam)
{
if (renderingcam == cam)
{
OnPreRender();
}
}
#endif
///
/// Where various image processing effects are applied, including the green screen effect itself.
///
private void OnPreRender()
{
if (screenManager.TextureEye == null || screenManager.TextureEye.width == 0) return;
if (canal.Equals(CANAL.FOREGROUND)) return;
RenderTexture tempAlpha = RenderTexture.GetTemporary(finalTexture.width, finalTexture.height, 0, RenderTextureFormat.RFloat, RenderTextureReadWrite.Linear);
RenderTexture tempFinalAlpha = RenderTexture.GetTemporary(finalTexture.width, finalTexture.height, 0, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Linear);
Graphics.Blit(screenManager.TextureEye, tempAlpha, greenScreenMat);
//preprocessMat.SetTexture("_MaskTex", tempAlpha);
preprocessMat.SetTexture(maskTexID, tempAlpha);
Graphics.Blit(screenManager.TextureEye, tempFinalAlpha, preprocessMat);
//If the sigma has changed, recompute the weights and offsets used by the blur.
if (sigma_ == 0)
{
if (sigma_ != currentSigma)
{
currentSigma = sigma_;
ZEDPostProcessingTools.ComputeWeights(currentSigma, out weights_, out offsets_);
//Send the values to the current shader
//blurMaterial.SetFloatArray("weights", weights_);
blurMaterial.SetFloatArray(weightsID, weights_);
//blurMaterial.SetFloatArray("offset", offsets_);
blurMaterial.SetFloatArray(offsetID, offsets_);
}
ZEDPostProcessingTools.Blur(tempFinalAlpha, finalTexture, blurMaterial, 0, 1, 1);
}
else
{
Graphics.Blit(tempFinalAlpha, finalTexture);
}
//Destroy all the temporary buffers
RenderTexture.ReleaseTemporary(tempAlpha);
RenderTexture.ReleaseTemporary(tempFinalAlpha);
}
///
/// Gets a value within a gaussian spread defined by sigma.
///
float Gaussian(float x, float sigma)
{
return (1.0f / (2.0f * Mathf.PI * sigma)) * Mathf.Exp(-((x * x) / (2.0f * sigma)));
}
///
/// Watch for changes in LastAccess and LastWrite times, and the renaming of files or directories.
/// Used to make sure the path to the config file remains valid.
///
///
public void CreateFileWatcher(string path)
{
if (!File.Exists(pathFileConfig)) return;
FileSystemWatcher watcher = new FileSystemWatcher();
watcher.Path = path;
watcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.DirectoryName;
watcher.Filter = Path.GetFileName(pathFileConfig);
watcher.Changed += new FileSystemEventHandler(OnChanged);
watcher.EnableRaisingEvents = true;
}
///
/// Event handler for when the path to the config file changes while within the editor.
///
private void OnChanged(object source, FileSystemEventArgs e)
{
toUpdateConfig = true;
}
}