Rollup merge of #103689 - saethlin:libtest-startup, r=thomcc

Do fewer passes and generally be more efficient when filtering tests

Follow-on of the work I started with this PR: https://github.com/rust-lang/rust/pull/99939

Basically, the startup code for libtest is really inefficient, but that's not usually a problem because it is distributed in release and workloads are small. But under Miri which can be 100x slower than a debug build, these inefficiencies explode.

Most of the diff here is making test filtering single-pass. There are a few other small optimizations as well, but they are more straightforward.

With this PR, the startup time of the `iced` tests with `--features=code_asm,mvex` drops from 17 to 2 minutes (I think Miri has gotten slower under this workload since #99939). The easiest way to try this out is to set `MIRI_LIB_SRC` to a checkout of this branch when running `cargo +nightly miri test --features=code_asm,mvex`.

r? `@thomcc`

library/test/src/console.rs

@@ -228,9 +228,9 @@ fn on_test_event(
     out: &mut dyn OutputFormatter,
 ) -> io::Result<()> {
     match (*event).clone() {
-        TestEvent::TeFiltered(ref filtered_tests, shuffle_seed) => {
+        TestEvent::TeFiltered(filtered_tests, shuffle_seed) => {
-            st.total = filtered_tests.len();
+            st.total = filtered_tests;
-            out.write_run_start(filtered_tests.len(), shuffle_seed)?;
+            out.write_run_start(filtered_tests, shuffle_seed)?;
         }
         TestEvent::TeFilteredOut(filtered_out) => {
             st.filtered_out = filtered_out;

library/test/src/event.rs

@@ -28,7 +28,7 @@ impl CompletedTest {
 
 #[derive(Debug, Clone)]
 pub enum TestEvent {
-    TeFiltered(Vec<TestDesc>, Option<u64>),
+    TeFiltered(usize, Option<u64>),
     TeWait(TestDesc),
     TeResult(CompletedTest),
     TeTimeout(TestDesc),

library/test/src/lib.rs

@@ -219,6 +219,35 @@ pub fn assert_test_result<T: Termination>(result: T) -> Result<(), String> {
     }
 }
 
+struct FilteredTests {
+    tests: Vec<(TestId, TestDescAndFn)>,
+    benchs: Vec<(TestId, TestDescAndFn)>,
+    next_id: usize,
+}
+
+impl FilteredTests {
+    fn add_bench(&mut self, desc: TestDesc, testfn: TestFn) {
+        let test = TestDescAndFn { desc, testfn };
+        self.benchs.push((TestId(self.next_id), test));
+        self.next_id += 1;
+    }
+    fn add_test(&mut self, desc: TestDesc, testfn: TestFn) {
+        let test = TestDescAndFn { desc, testfn };
+        self.tests.push((TestId(self.next_id), test));
+        self.next_id += 1;
+    }
+    fn add_bench_as_test(
+        &mut self,
+        desc: TestDesc,
+        benchfn: impl Fn(&mut Bencher) -> Result<(), String> + Send + 'static,
+    ) {
+        let testfn = DynTestFn(Box::new(move || {
+            bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
+        }));
+        self.add_test(desc, testfn);
+    }
+}
+
 pub fn run_tests<F>(
     opts: &TestOpts,
     tests: Vec<TestDescAndFn>,
@@ -247,45 +276,51 @@ where
 
     let tests_len = tests.len();
 
-    let mut filtered_tests = filter_tests(opts, tests);
+    let mut filtered = FilteredTests { tests: Vec::new(), benchs: Vec::new(), next_id: 0 };
-    if !opts.bench_benchmarks {
+
-        filtered_tests = convert_benchmarks_to_tests(filtered_tests);
+    for test in filter_tests(opts, tests) {
+        let mut desc = test.desc;
+        desc.name = desc.name.with_padding(test.testfn.padding());
+
+        match test.testfn {
+            DynBenchFn(benchfn) => {
+                if opts.bench_benchmarks {
+                    filtered.add_bench(desc, DynBenchFn(benchfn));
+                } else {
+                    filtered.add_bench_as_test(desc, benchfn);
+                }
+            }
+            StaticBenchFn(benchfn) => {
+                if opts.bench_benchmarks {
+                    filtered.add_bench(desc, StaticBenchFn(benchfn));
+                } else {
+                    filtered.add_bench_as_test(desc, benchfn);
+                }
+            }
+            testfn => {
+                filtered.add_test(desc, testfn);
+            }
+        };
     }
 
-    let filtered_tests = {
+    let filtered_out = tests_len - filtered.tests.len();
-        let mut filtered_tests = filtered_tests;
-        for test in filtered_tests.iter_mut() {
-            test.desc.name = test.desc.name.with_padding(test.testfn.padding());
-        }
-
-        filtered_tests
-    };
-
-    let filtered_out = tests_len - filtered_tests.len();
     let event = TestEvent::TeFilteredOut(filtered_out);
     notify_about_test_event(event)?;
 
-    let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
-
     let shuffle_seed = get_shuffle_seed(opts);
 
-    let event = TestEvent::TeFiltered(filtered_descs, shuffle_seed);
+    let event = TestEvent::TeFiltered(filtered.tests.len(), shuffle_seed);
     notify_about_test_event(event)?;
 
-    let (mut filtered_tests, filtered_benchs): (Vec<_>, _) = filtered_tests
-        .into_iter()
-        .enumerate()
-        .map(|(i, e)| (TestId(i), e))
-        .partition(|(_, e)| matches!(e.testfn, StaticTestFn(_) | DynTestFn(_)));
-
     let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
 
+    let mut remaining = filtered.tests;
     if let Some(shuffle_seed) = shuffle_seed {
-        shuffle_tests(shuffle_seed, &mut filtered_tests);
+        shuffle_tests(shuffle_seed, &mut remaining);
     }
     // Store the tests in a VecDeque so we can efficiently remove the first element to run the
     // tests in the order they were passed (unless shuffled).
-    let mut remaining = VecDeque::from(filtered_tests);
+    let mut remaining = VecDeque::from(remaining);
     let mut pending = 0;
 
     let (tx, rx) = channel::<CompletedTest>();
@@ -402,7 +437,7 @@ where
 
     if opts.bench_benchmarks {
         // All benchmarks run at the end, in serial.
-        for (id, b) in filtered_benchs {
+        for (id, b) in filtered.benchs {
             let event = TestEvent::TeWait(b.desc.clone());
             notify_about_test_event(event)?;
             run_test(opts, false, id, b, run_strategy, tx.clone(), Concurrent::No);
@@ -432,7 +467,9 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescA
     }
 
     // Skip tests that match any of the skip filters
-    filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
+    if !opts.skip.is_empty() {
+        filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
+    }
 
     // Excludes #[should_panic] tests
     if opts.exclude_should_panic {