From 6201361b8c80a2096a159584a818d57a8694d921 Mon Sep 17 00:00:00 2001 From: lisyarus Date: Sun, 6 Sep 2026 14:40:55 +0300 Subject: [PATCH] Add softplus and squareplus activation functions to ml library (idk why they're just neat lol) --- .../psemek/ml/neural_net/activation.hpp | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/libs/ml/include/psemek/ml/neural_net/activation.hpp b/libs/ml/include/psemek/ml/neural_net/activation.hpp index 62438eb5..415bd760 100644 --- a/libs/ml/include/psemek/ml/neural_net/activation.hpp +++ b/libs/ml/include/psemek/ml/neural_net/activation.hpp @@ -4,7 +4,6 @@ #include #include -#include #include namespace psemek::ml @@ -18,6 +17,8 @@ namespace psemek::ml (sigmoid) (tanh) (relu) + (softplus) + (squareplus) ) template @@ -27,14 +28,21 @@ namespace psemek::ml case activation_type::id: return x; case activation_type::sigmoid: - return 1.f / (1.f + std::exp(-x)); + return T{1} / (T{1} + std::exp(-x)); case activation_type::tanh: - return 2.f / (1.f + std::exp(- 2.f * x)) - 1.f; + return T{2} / (T{1} + std::exp(- T{2} * x)) - T{1}; case activation_type::relu: return std::max(T{0}, x); - default: - throw util::unknown_enum_value_exception{type}; + case activation_type::softplus: + return (x < T{0}) + ? std::log(T{1} + std::exp(x)) + : x + std::log(T{1} + std::exp(-1)); + case activation_type::squareplus: + return (x < T{0}) + ? T{2} / (std::sqrt(x * x + T{4}) - x) + : (x + std::sqrt(x * x + T{4})) / T{2}; } + throw util::unknown_enum_value_exception{type}; } template @@ -49,9 +57,15 @@ namespace psemek::ml return T{1} - value * value; case activation_type::relu: return value == T{0} ? T{0} : T{1}; - default: - throw util::unknown_enum_value_exception{type}; + case activation_type::softplus: + return T{1} - std::exp(- value); + case activation_type::squareplus: + { + auto v2 = value * value; + return v2 / (T{1} + v2); + } } + throw util::unknown_enum_value_exception{type}; } }