skip @inferred tests for PkgEval (#165)

They can also be manually disabled by setting `CHAINRULES_TEST_INFERRED`
to `false`. I have tested that all tests still pass with `JULIA_PKGEVAL`
set to `true`, do we need to test for that by spawning a new Julia
process? Once this is merged, I plan to replace occurences of
`@inferred` in ChainRules with `@maybe_inferred` as well, therefore I
exported that macro.

Co-authored-by: Lyndon White <oxinabox@ucc.asn.au>

Author: simeonschaub

docs/src/index.md

@@ -147,6 +147,36 @@ Test Failed at REPL[52]:1
 ERROR: There was an error during testing
 ```
 which should have passed the test.
+
+## Inference tests
+
+By default, all functions for testing rules check whether the output type (as well as that of the pullback for `rrule`s) can be completely inferred, such that everything is type stable:
+
+```jldoctest ex
+julia> function ChainRulesCore.rrule(::typeof(abs), x)
+           abs_pullback(Δ) = (NoTangent(), x >= 0 ? Δ : big(-1.0) * Δ)
+           return abs(x), abs_pullback
+       end
+
+julia> test_rrule(abs, 1.)
+test_rrule: abs on Float64: Error During Test at /home/runner/work/ChainRulesTestUtils.jl/ChainRulesTestUtils.jl/src/testers.jl:170
+  Got exception outside of a @test
+  return type Tuple{ChainRulesCore.NoTangent, Float64} does not match inferred return type Tuple{ChainRulesCore.NoTangent, Union{Float64, BigFloat}}
+[...]
+```
+
+This can be disabled on a per-rule basis using the `check_inferred` keyword argument:
+
+```jldoctest ex
+julia> test_rrule(abs, 1.; check_inferred=false)
+Test Summary:              | Pass  Total
+test_rrule: abs on Float64 |    5      5
+Test.DefaultTestSet("test_rrule: abs on Float64", Any[], 5, false, false)
+```
+
+This behavior can also be overridden globally by setting the environment variable `CHAINRULES_TEST_INFERRED` before ChainRulesTestUtils is loaded or by changing `ChainRulesTestUtils.TEST_INFERRED[]` from inside Julia.
+ChainRulesTestUtils can detect whether a test is run as part of [PkgEval](https://github.com/JuliaCI/PkgEval.jl)and in this case disables inference tests automatically. Packages can use [`@maybe_inferred`](@ref) to get the same behavior for other inference tests.
+
 # API Documentation
 
 ```@autodocs

src/ChainRulesTestUtils.jl

@@ -12,11 +12,22 @@ using Test
 import FiniteDifferences: rand_tangent
 
 const _fdm = central_fdm(5, 1; max_range=1e-2)
+const TEST_INFERRED = Ref(true)
 
 export TestIterator
 export test_approx, test_scalar, test_frule, test_rrule, generate_well_conditioned_matrix
 export ⊢
+export @maybe_inferred
 
+function __init__()
+    TEST_INFERRED[] = if haskey(ENV, "CHAINRULES_TEST_INFERRED")
+        parse(Bool, "CHAINRULES_TEST_INFERRED")
+    else
+        !parse(Bool, get(ENV, "JULIA_PKGEVAL", "false"))
+    end
+
+    !TEST_INFERRED[] && @warn "inference tests have been disabled"
+end
 
 include("generate_tangent.jl")
 include("data_generation.jl")

src/testers.jl

@@ -246,6 +246,17 @@ function _ensure_not_running_on_functor(f, name)
     end
 end
 
+"""
+    @maybe_inferred f(...)
+
+Like `@inferred`, but does not check the return type if tests are run as part of PkgEval or
+if the environment variable `CHAINRULES_TEST_INFERRED` is set to `false`.
+"""
+macro maybe_inferred(ex)
+    inferred = Expr(:macrocall, GlobalRef(Test, Symbol("@inferred")), __source__, ex)
+    return :(TEST_INFERRED[] ? $(esc(inferred)) : $(esc(ex)))
+end
+
 """
     _test_inferred(f, args...; kwargs...)
 
@@ -254,9 +265,9 @@ knowing how many `kwargs` there are.
 """
 function _test_inferred(f, args...; kwargs...)
     if isempty(kwargs)
-        @inferred f(args...)
+        @maybe_inferred f(args...)
     else
-        @inferred f(args...; kwargs...)
+        @maybe_inferred f(args...; kwargs...)
     end
 end
 

test/runtests.jl

@@ -5,6 +5,9 @@ using LinearAlgebra
 using Random
 using Test
 
+# in these meta tests, we always want to use `@inferred`
+ChainRulesTestUtils.TEST_INFERRED[] = true
+
 @testset "ChainRulesTestUtils.jl" begin
     include("meta_testing_tools.jl")
     include("iterator.jl")