added leakyrelu

Author: mxpoch

src/special/misc.jl

@@ -87,6 +87,7 @@ end
   pow_by_square(x, StaticInt(N))
 # @inline relu(x) = (y = zero(x); ifelse(x > y, x, y))
 @inline relu(x) = (y = zero(x); ifelse(x < y, y, x))
+@inline leakyrelu(x, a = 0.01) = (y = zero(x); ifelse(x < y, x*a, x))
 
 Base.sign(v::AbstractSIMD) = ifelse(v > 0, one(v), -one(v))
 

test/runtests.jl

@@ -1071,6 +1071,7 @@ include("testsetup.jl")
         trunc,
         round,
         VectorizationBase.relu,
+        VectorizationBase.leakyrelu,
         abs2,
         Base.FastMath.abs2_fast,
         Base.FastMath.sub_fast,