Use smaller threshold for ml gradient tests

libs/ml/include/psemek/ml/neural_net/activation.hpp

@@ -36,7 +36,7 @@ namespace psemek::ml
 		case activation_type::softplus:
 			return (x < T{0})
 				? std::log(T{1} + std::exp(x))
-				: x + std::log(T{1} + std::exp(-1));
+				: x + std::log(T{1} + std::exp(-x));
 		case activation_type::squareplus:
 			return (x < T{0})
 				? T{2} / (std::sqrt(x * x + T{4}) - x)

libs/ml/tests/neural_net/gradient.cpp

@@ -12,6 +12,27 @@ using namespace psemek::ml;
 using namespace psemek::random;
 using namespace psemek::math;
 
+test_case(ml_neural__net_activation)
+{
+	generator rng;
+	for (std::size_t i = 0; i < activation_type_values().size(); ++i)
+	{
+		auto a = static_cast<activation_type>(i);
+
+		for (std::size_t j = 0; j < 64; ++j)
+		{
+			double const eps = 1e-6;
+
+			double x = uniform(rng, -10.0, 10.0);
+
+			double value = activation(x, a);
+			double exact_derivative = activation_derivative(value, a);
+			double approx_derivative = (activation(x + eps, a) - value) / eps;
+			expect_close(exact_derivative, approx_derivative, 1e-6);
+		}
+	}
+}
+
 test_case(ml_neural__net_gradient)
 {
 	generator rng;
@@ -54,7 +75,7 @@ test_case(ml_neural__net_gradient)
 			nn.weights()[i] = old;
 
 			double numeric_gradient = (v1 - v0) / 2.0 / eps;
-			expect_close(numeric_gradient, learner.gradient()[i], 1e-4);
+			expect_close(numeric_gradient, learner.gradient()[i], 1e-6);
 		}
 	}
 }
@@ -101,7 +122,7 @@ test_case(ml_neural__net_arg__gradient)
 			input[i] = old;
 
 			double numeric_gradient = (v1 - v0) / 2.0 / eps;
-			expect_close(numeric_gradient, learner.arg_gradient()[i], 1e-4);
+			expect_close(numeric_gradient, learner.arg_gradient()[i], 1e-6);
 		}
 	}
 }