From 484a5809ef68259639a63653db5f6107d7ff86c1 Mon Sep 17 00:00:00 2001
From: lisyarus <lisyarus@gmail.com>
Date: Fri, 8 Apr 2022 11:59:52 +0300
Subject: [PATCH] Fix ui::painter_impl stencil implementation + support 255
 recursive stencil levels

---
 libs/ui/source/painter_impl.cpp | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/libs/ui/source/painter_impl.cpp b/libs/ui/source/painter_impl.cpp
index 17b91771..c416b80d 100644
--- a/libs/ui/source/painter_impl.cpp
+++ b/libs/ui/source/painter_impl.cpp
@@ -4,6 +4,7 @@
 #include <psemek/gfx/mesh.hpp>
 
 #include <psemek/util/overload.hpp>
+#include <psemek/util/to_string.hpp>
 #include <psemek/log/log.hpp>
 
 #include <variant>
@@ -218,7 +219,7 @@ void main()
 		gfx::mesh textured_mesh;
 
 		int stencil_level = 0;
-		static constexpr int max_stencil_level = 8;
+		static constexpr int max_stencil_level = 255;
 		std::vector<geom::box<float, 2>> stencil_bbox;
 
 		geom::box<float, 2> draw_bbox;
@@ -389,7 +390,7 @@ void main()
 	void painter_impl::begin_stencil()
 	{
 		if (impl().stencil_level == impl().max_stencil_level)
-			throw std::runtime_error("Only 8 recursive stencil levels are supported");
+			throw std::runtime_error(util::to_string("Only ", impl().max_stencil_level, " recursive stencil levels are supported"));
 
 		auto & batch = impl().batch<stencil_batch>({});
 		if (impl().stencil_level == 0)
@@ -398,12 +399,12 @@ void main()
 		batch.enable_depth = false;
 
 		batch.func = gl::EQUAL;
-		batch.ref = 2 * (1 << impl().stencil_level) - 1;
-		batch.mask = (1 << impl().stencil_level) - 1;
+		batch.ref = impl().stencil_level;
+		batch.mask = -1;
 
 		batch.sfail = gl::KEEP;
 		batch.dfail = gl::KEEP;
-		batch.dpass = gl::REPLACE;
+		batch.dpass = gl::INCR;
 
 		++impl().stencil_level;
 		impl().draw_bbox = {};
@@ -416,8 +417,8 @@ void main()
 		batch.enable_depth = true;
 
 		batch.func = gl::EQUAL;
-		batch.ref = (1 << impl().stencil_level) - 1;
-		batch.mask = (1 << impl().stencil_level) - 1;
+		batch.ref = impl().stencil_level;
+		batch.mask = -1;
 
 		batch.sfail = gl::KEEP;
 		batch.dfail = gl::KEEP;
@@ -428,14 +429,16 @@ void main()
 
 	void painter_impl::end_stencil()
 	{
+		--impl().stencil_level;
+
 		{
 			auto & clear_batch = impl().batch<stencil_batch>({});
 			clear_batch.enable_color = false;
 			clear_batch.enable_depth = false;
 
-			clear_batch.func = gl::EQUAL;
-			clear_batch.ref = (1 << impl().stencil_level) / 2 - 1;
-			clear_batch.mask = (1 << impl().stencil_level) / 2 - 1;
+			clear_batch.func = gl::GEQUAL;
+			clear_batch.ref = impl().stencil_level;
+			clear_batch.mask = -1;
 
 			clear_batch.sfail = gl::KEEP;
 			clear_batch.dfail = gl::KEEP;
@@ -446,22 +449,20 @@ void main()
 
 		{
 			auto & batch = impl().batch<stencil_batch>({});
-			if (impl().stencil_level == 1)
+			if (impl().stencil_level == 0)
 				batch.enable_stencil = false;
 			batch.enable_color = true;
 			batch.enable_depth = true;
 
-			batch.func = gl::NEVER;
-			batch.ref = 0;
-			batch.mask = 0;
+			batch.func = gl::EQUAL;
+			batch.ref = impl().stencil_level;
+			batch.mask = -1;
 
 			batch.sfail = gl::KEEP;
 			batch.dfail = gl::KEEP;
 			batch.dpass = gl::KEEP;
 		}
 
-		--impl().stencil_level;
-
 		impl().stencil_bbox.pop_back();
 	}