Compute a summary.

parent 6b7d6818
......@@ -23,7 +23,7 @@ pub use sop::Sop;
pub const MAXIMUM_ARTIFACT_SIZE: usize = 50_000;
/// Backends supported by the test suite.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Implementation {
Sop(String),
}
......@@ -46,7 +46,7 @@ impl serde::Serialize for Implementation {
}
/// (Backend, Version)-tuple supporting multiple versions per backend.
#[derive(Debug, Clone, serde::Serialize)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize)]
pub struct Version {
pub implementation: Implementation,
pub version: String,
......
......@@ -478,6 +478,18 @@ pub struct Verification {
pub comment: String,
}
impl Verification {
pub fn expect_timestamp<T>(&self, _t: T)
-> Result<()>
where T: Into<DateTime<Utc>>
{
unimplemented!()
}
pub fn summary(&self) -> Data {
format!("Good signature from {:X}", self.cert).into_bytes().into()
}
}
impl std::str::FromStr for Verification {
type Err = SOPError;
fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
......
......@@ -10,6 +10,8 @@ use crate::{
tests::{
Test,
TestMatrix,
Summary,
Scores,
},
};
......@@ -88,6 +90,7 @@ impl<'a> Report<'a> {
let mut toc = Vec::new();
let mut body = String::new();
let mut summary = Summary::default();
for (section, section_results) in results {
body.push_str(&section.render_section()?);
......@@ -96,6 +99,7 @@ impl<'a> Report<'a> {
let r = maybe_result?;
toc_section.push(Entry::new(&r.title()));
body.push_str(&r.render()?);
r.summarize(&mut summary);
}
toc.push((section, toc_section));
}
......@@ -107,6 +111,7 @@ impl<'a> Report<'a> {
title: format!("OpenPGP interoperability test suite"),
toc,
body,
summary: summary.for_rendering(),
configuration: self.configuration,
})
}
......@@ -121,6 +126,7 @@ pub struct Results<'a> {
title: String,
toc: Vec<(Entry, Vec<Entry>)>,
body: String,
summary: Vec<(String, Scores)>,
configuration: &'a Config,
}
......
use std::{
collections::HashMap,
};
use sequoia_openpgp as openpgp;
use openpgp::policy::StandardPolicy;
......@@ -247,6 +251,36 @@ impl TestMatrix {
pub fn title(&self) -> String {
self.title.clone()
}
pub fn summarize(&self, summary: &mut Summary) {
for (i, imp) in self.consumers.iter().enumerate() {
let mut good = 0;
let mut bad = 0;
for row in &self.results {
// Get the result corresponding to implementation
// 'imp'.
if let Some(r) = row.results.get(i) {
match r.score {
None => (),
Some(true) => good += 1,
Some(false) => bad += 1,
}
}
}
// Count all rows where we have an expectation, and the
// producer produced an artifact.
let have_expectations = self.results.iter().filter(|row| {
row.expectation.is_some() && ! row.results.is_empty()
}).count();
// Did it pass on all test vectors?
let all_good = good == have_expectations;
summary.add(imp.clone(), good, bad, all_good);
}
}
}
#[derive(Debug, serde::Serialize)]
......@@ -256,6 +290,53 @@ struct TestResults {
expectation: Option<Expectation>,
}
#[derive(Debug, Default, serde::Serialize)]
pub struct Summary {
score: HashMap<Version, Scores>,
}
impl Summary {
fn add(&mut self, imp: Version, good: usize, bad: usize, all_good: bool) {
let e = self.score.entry(imp).or_default();
e.good += good;
e.bad += bad;
if all_good {
e.all_good += 1;
}
}
/// Transforms the summary into a map suitable for rendering.
pub fn for_rendering(self) -> Vec<(String, Scores)> {
let mut r: Vec<(String, Scores)> =
self.score.into_iter()
.map(|(k, v)| (k.to_string(), v))
.collect();
r.sort_unstable_by(|a, b| a.1.cmp(&b.1).reverse());
r
}
}
#[derive(Debug, Default, PartialEq, Eq, serde::Serialize)]
pub struct Scores {
good: usize,
bad: usize,
all_good: usize,
}
impl Ord for Scores {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.all_good.cmp(&other.all_good)
.then(self.good.cmp(&other.good))
.then(self.bad.cmp(&other.bad).reverse())
}
}
impl PartialOrd for Scores {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
/// Extracts the public certificate from the given key.
pub fn extract_cert(key: &[u8]) -> Result<Data> {
use openpgp::Packet;
......
......@@ -50,6 +50,13 @@
}
/* /Rotate table headings. */
/* Summary table. */
table.summary th, table.summary td {
text-align: left;
padding: 0.1em 1em;
}
/* /Summary table. */
pre { background-color: #eee; }
.score-good {
......@@ -314,6 +321,7 @@
{%- endfor -%}
</ol>
</li>
<li><a href="#summary">Test Summary</a></li>
<li><a href="#hall-of-fame">Hall of Fame</a></li>
<li><a href="#configuration">Configuration</a></li>
</ol>
......@@ -321,6 +329,44 @@
<a name="test-results"><h1>Test Results</h1></a>
{{body | safe}}
<hr />
<a name="summary"><h1>Test Summary</h1></a>
<p>
This table summarizes the above results. Reducing the wealth of
information to a set of numbers necessarily loses information,
so take them with a grain of salt. Nevertheless, these number
provide an indication to what degree an implementation agrees
with the expectations of this test suite.
</p>
<p>
The first number is the number of tests where the implementation
matched the test suite's expectations on all vectors of a given
test.
</p>
<p>
The second number is the number of test vectors where the
implementation matched the test suite's expectations.
</p>
<p>
The third number is the number of test vectors where the
implementation did not match the test suite's expectations.
</p>
<table class="summary">
<tr>
<th>Implementation</th>
<th>Matched all</th>
<th>Matched</th>
<th>Mismatched</th>
</tr>
{%- for row in summary -%}
<tr>
<td>{{ row.0 }}</td>
<td>{{ row.1.all_good }}</td>
<td>{{ row.1.good }}</td>
<td>{{ row.1.bad }}</td>
</tr>
{%- endfor -%}
</table>
<hr />
{% include "hall-of-fame.inc.html" %}
<hr />
<a name="configuration"><h1>Configuration</h1></a>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment