Merge pull request #379 from rust-lang/tgross35/ntests

Streamline the way that test iteration count is determined  (replace `NTESTS`)

Author: tgross35

.github/workflows/main.yml

@@ -113,6 +113,11 @@ jobs:
         rustup target add x86_64-unknown-linux-musl
         cargo generate-lockfile && ./ci/run-docker.sh ${{ matrix.target }}
 
+    - name: Print test logs if available
+      if: always()
+      run: if [ -f "target/test-log.txt" ]; then cat target/test-log.txt; fi
+      shell: bash
+
   clippy:
     name: Clippy
     runs-on: ubuntu-24.04

configure.rs

@@ -8,6 +8,7 @@ pub struct Config {
     pub manifest_dir: PathBuf,
     pub out_dir: PathBuf,
     pub opt_level: u8,
+    pub cargo_features: Vec<String>,
     pub target_arch: String,
     pub target_env: String,
     pub target_family: Option<String>,
@@ -22,11 +23,16 @@ impl Config {
         let target_features = env::var("CARGO_CFG_TARGET_FEATURE")
             .map(|feats| feats.split(',').map(ToOwned::to_owned).collect())
             .unwrap_or_default();
+        let cargo_features = env::vars()
+            .filter_map(|(name, _value)| name.strip_prefix("CARGO_FEATURE_").map(ToOwned::to_owned))
+            .map(|s| s.to_lowercase().replace("_", "-"))
+            .collect();
 
         Self {
             manifest_dir: PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()),
             out_dir: PathBuf::from(env::var("OUT_DIR").unwrap()),
             opt_level: env::var("OPT_LEVEL").unwrap().parse().unwrap(),
+            cargo_features,
             target_arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(),
             target_env: env::var("CARGO_CFG_TARGET_ENV").unwrap(),
             target_family: env::var("CARGO_CFG_TARGET_FAMILY").ok(),
@@ -45,6 +51,7 @@ pub fn emit_libm_config(cfg: &Config) {
     emit_arch_cfg();
     emit_optimization_cfg(cfg);
     emit_cfg_shorthands(cfg);
+    emit_cfg_env(cfg);
     emit_f16_f128_cfg(cfg);
 }
 
@@ -53,6 +60,7 @@ pub fn emit_libm_config(cfg: &Config) {
 pub fn emit_test_config(cfg: &Config) {
     emit_optimization_cfg(cfg);
     emit_cfg_shorthands(cfg);
+    emit_cfg_env(cfg);
     emit_f16_f128_cfg(cfg);
 }
 
@@ -97,6 +105,13 @@ fn emit_cfg_shorthands(cfg: &Config) {
     }
 }
 
+/// Reemit config that we make use of for test logging.
+fn emit_cfg_env(cfg: &Config) {
+    println!("cargo:rustc-env=CFG_CARGO_FEATURES={:?}", cfg.cargo_features);
+    println!("cargo:rustc-env=CFG_OPT_LEVEL={}", cfg.opt_level);
+    println!("cargo:rustc-env=CFG_TARGET_FEATURES={:?}", cfg.target_features);
+}
+
 /// Configure whether or not `f16` and `f128` support should be enabled.
 fn emit_f16_f128_cfg(cfg: &Config) {
     println!("cargo:rustc-check-cfg=cfg(f16_enabled)");

crates/libm-test/benches/random.rs

@@ -2,8 +2,9 @@ use std::hint::black_box;
 use std::time::Duration;
 
 use criterion::{Criterion, criterion_main};
-use libm_test::gen::{CachedInput, random};
-use libm_test::{CheckBasis, CheckCtx, GenerateInput, MathOp, TupleCall};
+use libm_test::gen::random;
+use libm_test::gen::random::RandomInput;
+use libm_test::{CheckBasis, CheckCtx, MathOp, TupleCall};
 
 /// Benchmark with this many items to get a variety
 const BENCH_ITER_ITEMS: usize = if cfg!(feature = "short-benchmarks") { 50 } else { 500 };
@@ -47,7 +48,7 @@ macro_rules! musl_rand_benches {
 fn bench_one<Op>(c: &mut Criterion, musl_extra: MuslExtra<Op::CFn>)
 where
     Op: MathOp,
-    CachedInput: GenerateInput<Op::RustArgs>,
+    Op::RustArgs: RandomInput,
 {
     let name = Op::NAME;
 

crates/libm-test/src/gen.rs

@@ -1,74 +1,42 @@
 //! Different generators that can create random or systematic bit patterns.
 
-use crate::GenerateInput;
 pub mod domain_logspace;
 pub mod edge_cases;
 pub mod random;
 
-/// Helper type to turn any reusable input into a generator.
-#[derive(Clone, Debug, Default)]
-pub struct CachedInput {
-    pub inputs_f32: Vec<(f32, f32, f32)>,
-    pub inputs_f64: Vec<(f64, f64, f64)>,
-    pub inputs_i32: Vec<(i32, i32, i32)>,
+/// A wrapper to turn any iterator into an `ExactSizeIterator`. Asserts the final result to ensure
+/// the provided size was correct.
+#[derive(Debug)]
+pub struct KnownSize<I> {
+    total: u64,
+    current: u64,
+    iter: I,
 }
 
-impl GenerateInput<(f32,)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f32,)> {
-        self.inputs_f32.iter().map(|f| (f.0,))
+impl<I> KnownSize<I> {
+    pub fn new(iter: I, total: u64) -> Self {
+        Self { total, current: 0, iter }
     }
 }
 
-impl GenerateInput<(f32, f32)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f32, f32)> {
-        self.inputs_f32.iter().map(|f| (f.0, f.1))
-    }
-}
-
-impl GenerateInput<(i32, f32)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (i32, f32)> {
-        self.inputs_i32.iter().zip(self.inputs_f32.iter()).map(|(i, f)| (i.0, f.0))
-    }
-}
+impl<I: Iterator> Iterator for KnownSize<I> {
+    type Item = I::Item;
 
-impl GenerateInput<(f32, i32)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f32, i32)> {
-        GenerateInput::<(i32, f32)>::get_cases(self).map(|(i, f)| (f, i))
-    }
-}
+    fn next(&mut self) -> Option<Self::Item> {
+        let next = self.iter.next();
+        if next.is_some() {
+            self.current += 1;
+            return next;
+        }
 
-impl GenerateInput<(f32, f32, f32)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f32, f32, f32)> {
-        self.inputs_f32.iter().copied()
+        assert_eq!(self.current, self.total, "total items did not match expected");
+        None
     }
-}
 
-impl GenerateInput<(f64,)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f64,)> {
-        self.inputs_f64.iter().map(|f| (f.0,))
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let remaining = usize::try_from(self.total - self.current).unwrap();
+        (remaining, Some(remaining))
     }
 }
 
-impl GenerateInput<(f64, f64)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f64, f64)> {
-        self.inputs_f64.iter().map(|f| (f.0, f.1))
-    }
-}
-
-impl GenerateInput<(i32, f64)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (i32, f64)> {
-        self.inputs_i32.iter().zip(self.inputs_f64.iter()).map(|(i, f)| (i.0, f.0))
-    }
-}
-
-impl GenerateInput<(f64, i32)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f64, i32)> {
-        GenerateInput::<(i32, f64)>::get_cases(self).map(|(i, f)| (f, i))
-    }
-}
-
-impl GenerateInput<(f64, f64, f64)> for CachedInput {
-    fn get_cases(&self) -> impl Iterator<Item = (f64, f64, f64)> {
-        self.inputs_f64.iter().copied()
-    }
-}
+impl<I: Iterator> ExactSizeIterator for KnownSize<I> {}

crates/libm-test/src/gen/domain_logspace.rs

@@ -6,41 +6,26 @@ use libm::support::{IntTy, MinInt};
 
 use crate::domain::HasDomain;
 use crate::op::OpITy;
+use crate::run_cfg::{GeneratorKind, iteration_count};
 use crate::{CheckCtx, MathOp, logspace};
 
-/// Number of tests to run.
-// FIXME(ntests): replace this with a more logical algorithm
-const NTESTS: usize = {
-    if cfg!(optimizations_enabled) {
-        if crate::emulated()
-            || !cfg!(target_pointer_width = "64")
-            || cfg!(all(target_arch = "x86_64", target_vendor = "apple"))
-        {
-            // Tests are pretty slow on non-64-bit targets, x86 MacOS, and targets that run
-            // in QEMU.
-            100_000
-        } else {
-            5_000_000
-        }
-    } else {
-        // Without optimizations just run a quick check
-        800
-    }
-};
-
 /// Create a range of logarithmically spaced inputs within a function's domain.
 ///
 /// This allows us to get reasonably thorough coverage without wasting time on values that are
 /// NaN or out of range. Random tests will still cover values that are excluded here.
-pub fn get_test_cases<Op>(_ctx: &CheckCtx) -> impl Iterator<Item = (Op::FTy,)>
+pub fn get_test_cases<Op>(ctx: &CheckCtx) -> impl Iterator<Item = (Op::FTy,)>
 where
     Op: MathOp + HasDomain<Op::FTy>,
-    IntTy<Op::FTy>: TryFrom<usize>,
+    IntTy<Op::FTy>: TryFrom<u64>,
     RangeInclusive<IntTy<Op::FTy>>: Iterator,
 {
     let domain = Op::DOMAIN;
+    let ntests = iteration_count(ctx, GeneratorKind::Domain, 0);
+
+    // We generate logspaced inputs within a specific range, excluding values that are out of
+    // range in order to make iterations useful (random tests still cover the full range).
     let start = domain.range_start();
     let end = domain.range_end();
-    let steps = OpITy::<Op>::try_from(NTESTS).unwrap_or(OpITy::<Op>::MAX);
+    let steps = OpITy::<Op>::try_from(ntests).unwrap_or(OpITy::<Op>::MAX);
     logspace(start, end, steps).map(|v| (v,))
 }

crates/libm-test/src/gen/edge_cases.rs

@@ -3,18 +3,11 @@
 use libm::support::Float;
 
 use crate::domain::HasDomain;
+use crate::run_cfg::{check_near_count, check_point_count};
 use crate::{CheckCtx, FloatExt, MathOp};
 
-/// Number of values near an interesting point to check.
-// FIXME(ntests): replace this with a more logical algorithm
-const AROUND: usize = 100;
-
-/// Functions have infinite asymptotes, limit how many we check.
-// FIXME(ntests): replace this with a more logical algorithm
-const MAX_CHECK_POINTS: usize = 10;
-
 /// Create a list of values around interesting points (infinities, zeroes, NaNs).
-pub fn get_test_cases<Op, F>(_ctx: &CheckCtx) -> impl Iterator<Item = (F,)>
+pub fn get_test_cases<Op, F>(ctx: &CheckCtx) -> impl Iterator<Item = (F,)>
 where
     Op: MathOp<FTy = F> + HasDomain<F>,
     F: Float,
@@ -25,23 +18,26 @@ where
     let domain_start = domain.range_start();
     let domain_end = domain.range_end();
 
+    let check_points = check_point_count(ctx);
+    let near_points = check_near_count(ctx);
+
     // Check near some notable constants
-    count_up(F::ONE, values);
-    count_up(F::ZERO, values);
-    count_up(F::NEG_ONE, values);
-    count_down(F::ONE, values);
-    count_down(F::ZERO, values);
-    count_down(F::NEG_ONE, values);
+    count_up(F::ONE, near_points, values);
+    count_up(F::ZERO, near_points, values);
+    count_up(F::NEG_ONE, near_points, values);
+    count_down(F::ONE, near_points, values);
+    count_down(F::ZERO, near_points, values);
+    count_down(F::NEG_ONE, near_points, values);
     values.push(F::NEG_ZERO);
 
     // Check values near the extremes
-    count_up(F::NEG_INFINITY, values);
-    count_down(F::INFINITY, values);
-    count_down(domain_end, values);
-    count_up(domain_start, values);
-    count_down(domain_start, values);
-    count_up(domain_end, values);
-    count_down(domain_end, values);
+    count_up(F::NEG_INFINITY, near_points, values);
+    count_down(F::INFINITY, near_points, values);
+    count_down(domain_end, near_points, values);
+    count_up(domain_start, near_points, values);
+    count_down(domain_start, near_points, values);
+    count_up(domain_end, near_points, values);
+    count_down(domain_end, near_points, values);
 
     // Check some special values that aren't included in the above ranges
     values.push(F::NAN);
@@ -50,9 +46,9 @@ where
     // Check around asymptotes
     if let Some(f) = domain.check_points {
         let iter = f();
-        for x in iter.take(MAX_CHECK_POINTS) {
-            count_up(x, values);
-            count_down(x, values);
+        for x in iter.take(check_points) {
+            count_up(x, near_points, values);
+            count_down(x, near_points, values);
         }
     }
 
@@ -65,11 +61,11 @@ where
 
 /// Add `AROUND` values starting at and including `x` and counting up. Uses the smallest possible
 /// increments (1 ULP).
-fn count_up<F: Float>(mut x: F, values: &mut Vec<F>) {
+fn count_up<F: Float>(mut x: F, points: u64, values: &mut Vec<F>) {
     assert!(!x.is_nan());
 
     let mut count = 0;
-    while x < F::INFINITY && count < AROUND {
+    while x < F::INFINITY && count < points {
         values.push(x);
         x = x.next_up();
         count += 1;
@@ -78,11 +74,11 @@ fn count_up<F: Float>(mut x: F, values: &mut Vec<F>) {
 
 /// Add `AROUND` values starting at and including `x` and counting down. Uses the smallest possible
 /// increments (1 ULP).
-fn count_down<F: Float>(mut x: F, values: &mut Vec<F>) {
+fn count_down<F: Float>(mut x: F, points: u64, values: &mut Vec<F>) {
     assert!(!x.is_nan());
 
     let mut count = 0;
-    while x > F::NEG_INFINITY && count < AROUND {
+    while x > F::NEG_INFINITY && count < points {
         values.push(x);
         x = x.next_down();
         count += 1;