From 87fa2f427d044a120816f2bc83176bdb455ceccd Mon Sep 17 00:00:00 2001 From: josephjaspers Date: Fri, 22 Nov 2019 17:57:46 -0500 Subject: [PATCH] Add mish and softplus nonlinear functions --- include/neural_networks/layers/Nonlinear.h | 2 ++ include/operations/CMath.h | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/neural_networks/layers/Nonlinear.h b/include/neural_networks/layers/Nonlinear.h index 6c3d7c55..b3547230 100644 --- a/include/neural_networks/layers/Nonlinear.h +++ b/include/neural_networks/layers/Nonlinear.h @@ -31,6 +31,8 @@ auto funcName(SystemTag system, BC::size_t inputs) {\ BC_NONLINEAR_DEF(Tanh, tanh) BC_NONLINEAR_DEF(Logistic, logistic) BC_NONLINEAR_DEF(Relu, relu) +BC_NONLINEAR_DEF(SoftPlus, softplus) +BC_NONLINEAR_DEF(Mish, mish) #undef BC_NONLINEAR_DEF } diff --git a/include/operations/CMath.h b/include/operations/CMath.h index bb799466..32bf388c 100644 --- a/include/operations/CMath.h +++ b/include/operations/CMath.h @@ -157,9 +157,13 @@ BLACKCAT_FUNCTOR_DEF(Logistic, logistic, (1 / (1 + std::exp(-x))), BLACKCAT_FUNCTOR_DEF(Relu, relu, BC::traits::max(0, x), DERIVATIVE_DEF(x > 0 ? 1 : 0)); BLACKCAT_FUNCTOR_DEF(Logical, logical, x != 0 ? 1 : 0); - - - +BLACKCAT_FUNCTOR_DEF(SoftPlus, softplus, std::log(1 + std::exp(x)), DERIVATIVE_DEF(Logistic::apply(x))) +BLACKCAT_FUNCTOR_DEF(Mish, mish, + x * std::tanh(SoftPlus::apply(x)), + DERIVATIVE_DEF( + std::exp(x)* + (4*(x+1) + 4*(std::exp(2*x)) + std::exp(3*x) + std::exp(x)*(4*x+6)) / + std::pow((2*std::exp(x) + std::exp(2*x) + 2),2))) #undef BLACKCAT_FUNCTOR_DEF #undef BLACKCAT_MATH_DEF