Add ‘#[serial]’ tests (v2)

src/librustdoc/test.rs

@@ -490,6 +490,7 @@ impl Collector {
                 // compiler failures are test failures
                 should_panic: testing::ShouldPanic::No,
                 allow_fail: allow_fail,
+                serial: false,
             },
             testfn: testing::DynTestFn(box move |()| {
                 let panic = io::set_panic(None);

src/libsyntax/feature_gate.rs

@@ -369,6 +369,9 @@ declare_features! (
     // global allocators and their internals
     (active, global_allocator, "1.20.0", None),
     (active, allocator_internals, "1.20.0", None),
+
+    // Forces a test to execute in serial rather than in parallel with others.
+    (active, serial_tests, "1.20.0", Some(42684)),
 );
 
 declare_features! (
@@ -839,6 +842,11 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
                                  "allow_fail attribute is currently unstable",
                                  cfg_fn!(allow_fail))),
 
+    ("serial", Normal, Gated(Stability::Unstable,
+                             "serial_tests",
+                             "serial tests are currently unstable",
+                             cfg_fn!(serial_tests))),
+
     // Crate level attributes
     ("crate_name", CrateLevel, Ungated),
     ("crate_type", CrateLevel, Ungated),

src/libsyntax/test.rs

@@ -54,6 +54,7 @@ struct Test {
     ignore: bool,
     should_panic: ShouldPanic,
     allow_fail: bool,
+    serial: bool,
 }
 
 struct TestCtxt<'a> {
@@ -136,6 +137,7 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> {
                         ignore: is_ignored(&i),
                         should_panic: should_panic(&i, &self.cx),
                         allow_fail: is_allowed_fail(&i),
+                        serial: is_serial(&i),
                     };
                     self.cx.testfns.push(test);
                     self.tests.push(i.ident);
@@ -389,6 +391,10 @@ fn is_allowed_fail(i: &ast::Item) -> bool {
     i.attrs.iter().any(|attr| attr.check_name("allow_fail"))
 }
 
+fn is_serial(i: &ast::Item) -> bool {
+    i.attrs.iter().any(|attr| attr.check_name("serial"))
+}
+
 fn should_panic(i: &ast::Item, cx: &TestCtxt) -> ShouldPanic {
     match i.attrs.iter().find(|attr| attr.check_name("should_panic")) {
         Some(attr) => {
@@ -661,6 +667,7 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
     let should_panic_path = |name| {
         ecx.path(span, vec![self_id, test_id, ecx.ident_of("ShouldPanic"), ecx.ident_of(name)])
     };
+    let serial_expr = ecx.expr_bool(span, test.serial);
     let fail_expr = match test.should_panic {
         ShouldPanic::No => ecx.expr_path(should_panic_path("No")),
         ShouldPanic::Yes(msg) => {
@@ -683,8 +690,8 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
         vec![field("name", name_expr),
              field("ignore", ignore_expr),
              field("should_panic", fail_expr),
-             field("allow_fail", allow_fail_expr)]);
-
+             field("allow_fail", allow_fail_expr),
+             field("serial", serial_expr)]);
 
     let mut visible_path = match cx.toplevel_reexport {
         Some(id) => vec![id],

src/libtest/lib.rs

@@ -213,6 +213,7 @@ pub struct TestDesc {
     pub ignore: bool,
     pub should_panic: ShouldPanic,
     pub allow_fail: bool,
+    pub serial: bool,
 }
 
 #[derive(Clone)]
@@ -425,7 +426,10 @@ Test Attributes:
     #[ignore]      - When applied to a function which is already attributed as a
                      test, then the test runner will ignore these tests during
                      normal test runs. Running with --ignored will run these
-                     tests."#,
+                     tests.
+    #[serial]      - When applied to a function which is already attributed as a
+                     test, then the test runner will not run these tests in
+                     parallel with any other tests."#,
              usage = options.usage(&message));
 }
 
@@ -970,13 +974,15 @@ fn should_sort_failures_before_printing_them() {
         ignore: false,
         should_panic: ShouldPanic::No,
         allow_fail: false,
+        serial: false,
     };
 
     let test_b = TestDesc {
         name: StaticTestName("b"),
         ignore: false,
         should_panic: ShouldPanic::No,
         allow_fail: false,
+        serial: false,
     };
 
     let mut st = ConsoleTestState {
@@ -1091,7 +1097,9 @@ pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F)
         None => get_concurrency(),
     };
 
-    let mut remaining = filtered_tests;
+    let (serial, mut remaining): (Vec<_>, Vec<_>) = filtered_tests
+        .into_iter()
+        .partition(|t| t.desc.serial);
     remaining.reverse();
     let mut pending = 0;
 
@@ -1161,6 +1169,13 @@ pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F)
         pending -= 1;
     }
 
+    for test in serial {
+        callback(TeWait(test.desc.clone(), test.testfn.padding()))?;
+        run_test(opts, !opts.run_tests, test, tx.clone());
+        let (desc, result, stdout) = rx.recv().unwrap();
+        callback(TeResult(desc, result, stdout))?;
+    }
+
     if opts.bench_benchmarks {
         // All benchmarks run at the end, in serial.
         // (this includes metric fns)
@@ -1723,8 +1738,14 @@ pub mod bench {
 mod tests {
     use test::{TrFailed, TrFailedMsg, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc,
                TestDescAndFn, TestOpts, run_test, MetricMap, StaticTestName, DynTestName,
-               DynTestFn, ShouldPanic};
+               DynTestFn, ShouldPanic, TestResult};
+    use super::{TestEvent, run_tests};
+    use std::io;
     use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+    use std::thread::sleep;
+    use std::time::Duration;
+
     use bench;
     use Bencher;
 
@@ -1739,6 +1760,7 @@ mod tests {
                 ignore: true,
                 should_panic: ShouldPanic::No,
                 allow_fail: false,
+                serial: false,
             },
             testfn: DynTestFn(Box::new(move |()| f())),
         };
@@ -1757,6 +1779,7 @@ mod tests {
                 ignore: true,
                 should_panic: ShouldPanic::No,
                 allow_fail: false,
+                serial: false,
             },
             testfn: DynTestFn(Box::new(move |()| f())),
         };
@@ -1777,6 +1800,7 @@ mod tests {
                 ignore: false,
                 should_panic: ShouldPanic::Yes,
                 allow_fail: false,
+                serial: false,
             },
             testfn: DynTestFn(Box::new(move |()| f())),
         };
@@ -1797,6 +1821,7 @@ mod tests {
                 ignore: false,
                 should_panic: ShouldPanic::YesWithMessage("error message"),
                 allow_fail: false,
+                serial: false,
             },
             testfn: DynTestFn(Box::new(move |()| f())),
         };
@@ -1819,6 +1844,7 @@ mod tests {
                 ignore: false,
                 should_panic: ShouldPanic::YesWithMessage(expected),
                 allow_fail: false,
+                serial: false,
             },
             testfn: DynTestFn(Box::new(move |()| f())),
         };
@@ -1837,6 +1863,7 @@ mod tests {
                 ignore: false,
                 should_panic: ShouldPanic::Yes,
                 allow_fail: false,
+                serial: false,
             },
             testfn: DynTestFn(Box::new(move |()| f())),
         };
@@ -1871,6 +1898,7 @@ mod tests {
                                  ignore: true,
                                  should_panic: ShouldPanic::No,
                                  allow_fail: false,
+                                 serial: false,
                              },
                              testfn: DynTestFn(Box::new(move |()| {})),
                          },
@@ -1880,6 +1908,7 @@ mod tests {
                                  ignore: false,
                                  should_panic: ShouldPanic::No,
                                  allow_fail: false,
+                                 serial: false,
                              },
                              testfn: DynTestFn(Box::new(move |()| {})),
                          }];
@@ -1904,6 +1933,7 @@ mod tests {
                     ignore: false,
                     should_panic: ShouldPanic::No,
                     allow_fail: false,
+                    serial: false,
                 },
                 testfn: DynTestFn(Box::new(move |()| {}))
             })
@@ -1986,6 +2016,7 @@ mod tests {
                         ignore: false,
                         should_panic: ShouldPanic::No,
                         allow_fail: false,
+                        serial: false,
                     },
                     testfn: DynTestFn(Box::new(move |()| testfn())),
                 };
@@ -2010,6 +2041,118 @@ mod tests {
         }
     }
 
+    fn panic_on_non_ok_result(result: &TestResult) -> io::Result<()> {
+        match *result {
+            TestResult::TrOk => Ok(()),
+            TestResult::TrFailed => panic!("test failed (no message)"),
+            TestResult::TrFailedMsg(ref s) => panic!("test failed: {}", s),
+            TestResult::TrIgnored => panic!("test ignored"),
+            _ => panic!("test returned unexpected result"),
+        }
+    }
+
+    #[test]
+    pub fn stress_test_serial_tests() {
+        let mut opts = TestOpts::new();
+        opts.run_tests = true;
+        opts.test_threads = Some(2);
+
+        let lock = Arc::new(RwLock::new(0));
+        let lock2 = lock.clone();
+        let lock3 = lock.clone();
+
+        let tests = vec![
+            TestDescAndFn {
+                desc: TestDesc {
+                    name: StaticTestName("first"),
+                    ignore: false,
+                    should_panic: ShouldPanic::No,
+                    allow_fail: false,
+                    serial: true,
+                },
+                testfn: DynTestFn(Box::new(move |()| {
+                    let mut c = lock2.write().unwrap();
+                    sleep(Duration::from_millis(200));
+                    *c += 1;
+                    assert_eq!(*c, 1);
+                }))
+            },
+            TestDescAndFn {
+                desc: TestDesc {
+                    name: StaticTestName("second"),
+                    ignore: false,
+                    should_panic: ShouldPanic::No,
+                    allow_fail: false,
+                    serial: true,
+                },
+                testfn: DynTestFn(Box::new(move |()| {
+                    let mut c = lock3.write().unwrap();
+                    assert_eq!(*c, 1);
+                    *c += 1;
+                }))
+            }
+        ];
+
+        run_tests(&opts, tests, |e| {
+            match e {
+                TestEvent::TeFilteredOut(n) if n > 0 => panic!("filtered out"),
+                TestEvent::TeTimeout(_) => panic!("timeout"),
+                TestEvent::TeResult(_, ref result, _) =>
+                    panic_on_non_ok_result(result),
+                _ => Ok(())
+            }
+        }).unwrap();
+
+        assert_eq!(*(*lock).read().unwrap(), 2);
+    }
+
+    #[test]
+    pub fn run_concurrent_tests_concurrently() {
+        let mut opts = TestOpts::new();
+        opts.run_tests = true;
+        opts.test_threads = Some(2);
+
+        let (tx, rx) = channel::<()>();
+
+        let tests = vec![
+            TestDescAndFn {
+                desc: TestDesc {
+                    name: StaticTestName("first"),
+                    ignore: false,
+                    should_panic: ShouldPanic::No,
+                    allow_fail: false,
+                    serial: false,
+                },
+                testfn: DynTestFn(Box::new(move |()| {
+                    rx.recv_timeout(Duration::from_secs(1)).unwrap();
+                }))
+            },
+            TestDescAndFn {
+                desc: TestDesc {
+                    name: StaticTestName("second"),
+                    ignore: false,
+                    should_panic: ShouldPanic::No,
+                    allow_fail: false,
+                    serial: false,
+                },
+                testfn: DynTestFn(Box::new(move |()| {
+                    sleep(Duration::from_millis(100));
+                    tx.send(()).unwrap();
+                }))
+            },
+        ];
+
+        run_tests(&opts, tests, |e| {
+            match e {
+                TestEvent::TeFilteredOut(n) if n > 0 => panic!("filtered out"),
+                TestEvent::TeTimeout(_) => panic!("timeout"),
+                TestEvent::TeResult(_, ref result, _) =>
+                    panic_on_non_ok_result(result),
+                _ => Ok(())
+            }
+        }).unwrap();
+    }
+
     #[test]
     pub fn test_metricmap_compare() {
         let mut m1 = MetricMap::new();

src/test/compile-fail/feature-gate-serial_tests.rs

@@ -0,0 +1,16 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// check that #[serial] is feature-gated
+
+#[serial] //~ ERROR serial tests are currently unstable
+fn in_serial() {
+    assert!(true);
+}

src/tools/compiletest/src/main.rs

@@ -475,6 +475,7 @@ pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn
             ignore: ignore,
             should_panic: should_panic,
             allow_fail: false,
+            serial: false,
         },
         testfn: make_test_closure(config, testpaths),
     }