yggdrasil/ring
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Automatically print out failing test vectors; keep going on failure.

Browse Source 
Previously, "fail fast" semantics were implemented, where the first
failure stopped the entire test. Now, `test::from_file` keeps going
on failure. Also, when a test fails, the entire test case is printed
out automatically.
This commit is contained in:
Brian Smith 2016-07-03 12:37:49 -10:00
parent e0e63abebc
commit 15a61c8057
4 changed files with 194 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
src/test.rs
View File

@ -22,7 +22,9 @@
//! tests are the most complicated because they use named sections. Other tests
//! avoid named sections and so are easier to understand.
//!
//! # Example
//! # Examples
//!
//! ## Writing Tests
//!
//! Input files look like this:
//!
@ -65,6 +67,54 @@
//!
//! Note that `consume_digest_alg` automatically maps the string "SHA1" to a
//! reference to `digest::SHA1`, "SHA256" to `digest::SHA256`, etc.
//!
//! ## Output When a Test Fails
//!
//! When a test case fails, the framework automatically prints out the test
//! case. If the test case failed with a panic, then the backtrace of the panic
//! will be printed too. For example, let's say the failing test case looks
//! like this:
//!
//! ```text
//! Curve = P-256
//! a = 2b11cb945c8cf152ffa4c9c2b1c965b019b35d0b7626919ef0ae6cb9d232f8af
//! b = 18905f76a53755c679fb732b7762251075ba95fc5fedb60179e730d418a9143c
//! r = 18905f76a53755c679fb732b7762251075ba95fc5fedb60179e730d418a9143c
//! ```
//! If the test fails, this will be printed (if `$RUST_BACKTRACE` is `1`):
//!
//! ```text
//! src/example_tests.txt: Test panicked.
//! Curve = P-256
//! a = 2b11cb945c8cf152ffa4c9c2b1c965b019b35d0b7626919ef0ae6cb9d232f8af
//! b = 18905f76a53755c679fb732b7762251075ba95fc5fedb60179e730d418a9143c
//! r = 18905f76a53755c679fb732b7762251075ba95fc5fedb60179e730d418a9143c
//! thread 'example_test' panicked at 'Test failed.', src\test.rs:206
//! stack backtrace:
//! 0: 0x7ff654a05c7c - std::rt::lang_start::h61f4934e780b4dfc
//! 1: 0x7ff654a04f32 - std::rt::lang_start::h61f4934e780b4dfc
//! 2: 0x7ff6549f505d - std::panicking::rust_panic_with_hook::hfe203e3083c2b544
//! 3: 0x7ff654a0825b - rust_begin_unwind
//! 4: 0x7ff6549f63af - std::panicking::begin_panic_fmt::h484cd47786497f03
//! 5: 0x7ff654a07e9b - rust_begin_unwind
//! 6: 0x7ff654a0ae95 - core::panicking::panic_fmt::h257ceb0aa351d801
//! 7: 0x7ff654a0b190 - core::panicking::panic::h4bb1497076d04ab9
//! 8: 0x7ff65496dc41 - from_file<closure>
//! at C:\Users\Example\example\<core macros>:4
//! 9: 0x7ff65496d49c - example_test
//! at C:\Users\Example\example\src\example.rs:652
//! 10: 0x7ff6549d192a - test::stats::Summary::new::ha139494ed2e4e01f
//! 11: 0x7ff6549d51a2 - test::stats::Summary::new::ha139494ed2e4e01f
//! 12: 0x7ff654a0a911 - _rust_maybe_catch_panic
//! 13: 0x7ff6549d56dd - test::stats::Summary::new::ha139494ed2e4e01f
//! 14: 0x7ff654a03783 - std::sys::thread::Thread::new::h2b08da6cd2517f79
//! 15: 0x7ff968518101 - BaseThreadInitThunk
//! ```
//!
//! Notice that the output shows the name of the data file
//! (`src/example_tests.txt`), the test inputs that led to the failure, and the
//! stack trace to line in the test code that panicked: entry 9 in the stack
//! trace pointing to line 652 of the file `example.rs`.
use digest;
use std;
@ -76,7 +126,7 @@ use std::io::BufRead;
/// attribute in the test case must be consumed exactly once; this helps catch
/// typos and omissions.
pub struct TestCase {
attributes: std::collections::HashMap<String, String>,
attributes: Vec<(String, String, bool)>,
}
impl TestCase {
@ -140,7 +190,16 @@ impl TestCase {
/// Like `consume_string()` except it returns `None` if the test case
/// doesn't have the attribute.
pub fn consume_optional_string(&mut self, key: &str) -> Option<String> {
self.attributes.remove(key)
for &mut (ref name, ref value, ref mut consumed) in &mut self.attributes {
if key == name {
if *consumed {
panic!("Attribute {} was already consumed", key);
}
*consumed = true;
return Some(value.clone());
}
}
None
}
}
@ -156,20 +215,45 @@ pub fn from_file<F>(test_data_relative_file_path: &str, mut f: F)
let mut lines = std::io::BufReader::new(&file).lines();
let mut current_section = String::from("");
let mut failed = false;
loop {
match parse_test_case(&mut current_section, &mut lines) {
Some(ref mut test_case) => {
f(&current_section, test_case).unwrap();
let mut test_case =
match parse_test_case(&mut current_section, &mut lines) {
Some(test_case) => test_case,
None => { break; },
};
// Make sure all the attributes in the test case were consumed.
assert!(test_case.attributes.is_empty());
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
f(&current_section, &mut test_case)
}));
let result = match result {
Ok(Ok(())) => {
if !test_case.attributes.iter().any(
|&(_, _, ref consumed)| !consumed) {
Ok(())
} else {
failed = true;
Err("Test didn't consume all attributes.")
}
},
Ok(Err(_)) => Err("Test returned Err(())."),
Err(_) => Err("Test panicked."),
};
None => {
break;
if let Err(msg) = result {
failed = true;
println!("{}: {}", test_data_relative_file_path, msg);
for (ref name, ref value, ref consumed) in test_case.attributes {
let consumed_str = if *consumed { "" } else { " (unconsumed)" };
println!("{}{} = {}", name, consumed_str, value);
}
}
};
}
if failed {
panic!("Test failed.")
}
}
@ -206,7 +290,7 @@ type FileLines<'a> = std::io::Lines<std::io::BufReader<&'a std::fs::File>>;
fn parse_test_case(current_section: &mut String,
lines: &mut FileLines) -> Option<TestCase> {
let mut attributes = std::collections::HashMap::new();
let mut attributes = Vec::new();
let mut is_first_line = true;
loop {
@ -230,13 +314,17 @@ fn parse_test_case(current_section: &mut String,
// End of the file on a non-empty test cases ends the test case.
None => {
return Some(TestCase { attributes: attributes });
return Some(TestCase {
attributes: attributes,
});
},
// A blank line ends a test case if the test case isn't empty.
Some(ref line) if line.len() == 0 => {
if !is_first_line {
return Some(TestCase { attributes: attributes });
return Some(TestCase {
attributes: attributes,
});
}
// Ignore leading blank lines.
},
@ -265,9 +353,90 @@ fn parse_test_case(current_section: &mut String,
assert!(value.len() != 0);
// Checking is_none() ensures we don't accept duplicate keys.
assert!(attributes.insert(String::from(key),
String::from(value)).is_none());
attributes.push((String::from(key), String::from(value), false));
}
}
}
}
#[cfg(test)]
mod tests {
use test;
#[test]
fn one_ok() {
test::from_file("src/test_1_tests.txt", |_, test_case| {
let _ = test_case.consume_string("Key");
Ok(())
});
}
#[test]
#[should_panic(expected = "Test failed.")]
fn one_err() {
test::from_file("src/test_1_tests.txt", |_, test_case| {
let _ = test_case.consume_string("Key");
Err(())
});
}
#[test]
#[should_panic(expected = "Test failed.")]
fn one_panics() {
test::from_file("src/test_1_tests.txt", |_, test_case| {
let _ = test_case.consume_string("Key");
panic!("");
});
}
#[test] #[should_panic(expected = "Test failed.")] fn first_err() { err_one(0) }
#[test] #[should_panic(expected = "Test failed.")] fn middle_err() { err_one(1) }
#[test] #[should_panic(expected = "Test failed.")] fn last_err() { err_one(2) }
fn err_one(test_to_fail: usize) {
let mut n = 0;
test::from_file("src/test_3_tests.txt", |_, test_case| {
let _ = test_case.consume_string("Key");
let result = if n != test_to_fail {
Ok(())
} else {
Err(())
};
n += 1;
result
});
}
#[test] #[should_panic(expected = "Test failed.")] fn first_panic() { panic_one(0) }
#[test] #[should_panic(expected = "Test failed.")] fn middle_panic() { panic_one(1) }
#[test] #[should_panic(expected = "Test failed.")] fn last_panic() { panic_one(2) }
fn panic_one(test_to_fail: usize) {
let mut n = 0;
test::from_file("src/test_3_tests.txt", |_, test_case| {
let _ = test_case.consume_string("Key");
if n == test_to_fail {
panic!("Oh Noes!");
};
n += 1;
Ok(())
});
}
#[test]
#[should_panic]
fn syntax_error() {
test::from_file("src/test_1_syntax_error_tests.txt", |_, _| {
Ok(())
});
}
#[test]
#[should_panic]
fn file_not_found() {
test::from_file("src/test_file_not_found_tests.txt", |_, _| {
Ok(())
});
}
}

1
src/test_1_syntax_error_tests.txt Normal file
View File

@ -0,0 +1 @@
Key: 0

3
src/test_1_tests.txt Normal file
View File

@ -0,0 +1,3 @@
Key = Value

5
src/test_3_tests.txt Normal file
View File

@ -0,0 +1,5 @@
Key = 0
Key = 1
Key = 2