statistics.rs 13 KB
Newer Older
1 2
/// Collects statistics about the SKS packet dump using the openpgp
/// crate, Sequoia's low-level API.
3 4
///
/// Note that to achieve reasonable performance, you need to compile
5
/// Sequoia and this program with optimizations:
6
///
7 8
///     % cargo run -p sequoia-openpgp --example statistics --release \
///           -- <packet-dump>
9 10

use std::env;
11
extern crate sequoia_openpgp as openpgp;
Justus Winter's avatar
Justus Winter committed
12 13 14 15
use crate::openpgp::Packet;
use crate::openpgp::constants::SignatureType;
use crate::openpgp::packet::{user_attribute, BodyLength, Tag};
use crate::openpgp::parse::{Parse, PacketParserResult, PacketParser};
16 17 18

fn main() {
    let args: Vec<String> = env::args().collect();
19
    if args.len() < 2 {
20
        panic!("Collects statistics about OpenPGP packet dumps.\n\n\
21
                Usage: {} <packet-dump> [<packet-dump>...]\n", args[0]);
22 23 24 25 26 27 28 29 30 31 32 33 34
    }

    // Global stats.
    let mut packet_count = 0;
    let mut packet_size = 0 as usize;

    // Per-tag statistics.
    let mut tags_count = vec![0; 64];
    let mut tags_unknown = vec![0; 64];
    let mut tags_size_bytes = vec![0 as usize; 64];
    let mut tags_size_count = vec![0; 64];
    let mut tags_size_min = vec![::std::u32::MAX; 64];
    let mut tags_size_max = vec![0; 64];
35
    let mut sigs_count = vec![0; 256];
36

37 38 39 40 41 42 43 44
    // Signature Subpacket statistics.
    let mut sigs_subpacket_tags_count = vec![0; 256];
    let mut sigs_subpacket_tags_unknown = vec![0; 256];
    let mut sigs_subpacket_tags_size_bytes = vec![0 as usize; 256];
    let mut sigs_subpacket_tags_size_count = vec![0; 256];
    let mut sigs_subpacket_tags_size_min = vec![::std::u32::MAX; 256];
    let mut sigs_subpacket_tags_size_max = vec![0; 256];

45 46 47 48 49 50
    // Per-TPK statistics.
    let mut tpk_count = 0;
    let mut tpk = PerTPK::min();
    let mut tpk_min = PerTPK::max();
    let mut tpk_max = PerTPK::min();

51 52 53 54 55
    // UserAttribute statistics.
    let mut ua_image_count = vec![0; 256];
    let mut ua_unknown_count = vec![0; 256];
    let mut ua_invalid_count = 0;

56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
    // For each input file, create a parser.
    for input in &args[1..] {
        eprintln!("Parsing {}...", input);
        let mut ppr = PacketParser::from_file(input)
            .expect("Failed to create reader");

        // Iterate over all packets.
        while let PacketParserResult::Some(pp) = ppr {
            // While the packet is in the parser, get some data for later.
            let size = match pp.header().length {
                BodyLength::Full(n) => Some(n),
                _ => None,
            };

            // Get the packet and advance the parser.
            let (packet, tmp) = pp.next().expect("Failed to get next packet");
            ppr = tmp;

            packet_count += 1;
75
            if let Some(n) = size {
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
                packet_size += n as usize;
            }
            let i = u8::from(packet.tag()) as usize;
            tags_count[i] += 1;

            match packet {
                // If a new TPK starts, update TPK statistics.
                Packet::PublicKey(_) | Packet::SecretKey(_) => {
                    if tpk_count > 0 {
                        tpk.update_min_max(&mut tpk_min, &mut tpk_max);
                    }
                    tpk_count += 1;
                    tpk = PerTPK::min();
                },

                Packet::Signature(ref sig) => {
92 93
                    sigs_count[u8::from(sig.typ()) as usize] += 1;
                    tpk.sigs[u8::from(sig.typ()) as usize] += 1;
94 95 96 97

                    for (_offset, len, sub) in sig.hashed_area().iter()
                        .chain(sig.unhashed_area().iter())
                    {
Justus Winter's avatar
Justus Winter committed
98
                        use crate::openpgp::packet::signature::subpacket::*;
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
                        let i = u8::from(sub.tag) as usize;
                        sigs_subpacket_tags_count[i] += 1;
                        if let SubpacketValue::Unknown(_) = sub.value {
                            sigs_subpacket_tags_unknown
                                [u8::from(sub.tag) as usize] += 1;
                        } else {
                            sigs_subpacket_tags_size_bytes[i] += len;
                            sigs_subpacket_tags_size_count[i] += 1;
                            let len = len as u32;
                            if len < sigs_subpacket_tags_size_min[i] {
                                sigs_subpacket_tags_size_min[i] = len;
                            }
                            if len > sigs_subpacket_tags_size_max[i] {
                                sigs_subpacket_tags_size_max[i] = len;
                            }
                        }
                    }
116 117
                },

118
                Packet::UserAttribute(ref ua) => {
Justus Winter's avatar
Justus Winter committed
119 120
                    use crate::user_attribute::Subpacket;
                    use crate::user_attribute::Image;
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
                    for subpacket in ua.subpackets() {
                        match subpacket {
                            Ok(Subpacket::Image(i)) => match i {
                                Image::JPEG(_) =>
                                    ua_image_count[1] += 1,
                                Image::Private(n, _) =>
                                    ua_image_count[n as usize] += 1,
                                Image::Unknown(n, _) =>
                                    ua_image_count[n as usize] += 1,
                            },
                            Ok(Subpacket::Unknown(n, _)) =>
                                ua_unknown_count[n as usize] += 1,
                            Err(_) => ua_invalid_count += 1,
                        }
                    }
                },

138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
                _ => (),
            }

            if let Packet::Unknown(_) = packet {
                tags_unknown[i] += 1;
            } else {
                // Only record size statistics of packets we successfully
                // parsed.
                if let Some(n) = size {
                    tags_size_bytes[i] += n as usize;
                    tags_size_count[i] += 1;
                    if n < tags_size_min[i] {
                        tags_size_min[i] = n;
                    }
                    if n > tags_size_max[i] {
                        tags_size_max[i] = n;
                    }

                    tpk.bytes += n as usize;
157 158
                }

159 160
                tpk.packets += 1;
                tpk.tags[i] += 1;
161 162
            }
        }
163
        tpk.update_min_max(&mut tpk_min, &mut tpk_max);
164 165 166
    }

    // Print statistics.
167 168 169
    println!("# Packet statistics");
    println!();
    println!("{:>14} {:>9} {:>9} {:>9} {:>9} {:>9} {:>12}",
170
             "", "count", "unknown",
171
             "min size", "mean size", "max size", "sum size");
172
    println!("-------------------------------------------------------\
173
              -----------------------");
174

175 176 177
    for t in 0..64 {
        let count = tags_count[t];
        if count > 0 {
178
            println!("{:>14} {:>9} {:>9} {:>9} {:>9} {:>9} {:>12}",
179 180 181 182 183
                     format!("{:?}", Tag::from(t as u8)),
                     count,
                     tags_unknown[t],
                     tags_size_min[t],
                     tags_size_bytes[t] / tags_size_count[t],
184 185
                     tags_size_max[t],
                     tags_size_bytes[t]);
186 187 188
        }
    }

189 190 191 192 193 194 195 196 197 198 199 200 201 202
    if tags_count[u8::from(Tag::Signature) as usize] > 0 {
        println!();
        println!("# Signature statistics");
        println!();
        println!("{:>22} {:>9}",
                 "", "count",);
        println!("--------------------------------");
        for t in 0..256 {
            let max = tpk_max.sigs[t];
            if max > 0 {
                println!("{:>22} {:>9}",
                         format!("{:?}", SignatureType::from(t as u8)),
                         sigs_count[t]);
            }
203
        }
204 205 206 207 208 209 210 211 212 213 214 215 216 217

        println!();
        println!("# Signature Subpacket statistics");
        println!();
        println!("{:>30} {:>8} {:>6} {:>4} {:>4} {:>5} {:>14}",
                 "", "", "",
                 "min", "mean", "max", "sum");
        println!("{:>30} {:>8} {:>6} {:>4} {:>4} {:>5} {:>14}",
                 "", "#", "?",
                 "size", "size", "size", "size");
        println!("-------------------------------------------------------\
                  ----------------------");

        for t in 0..256 {
Justus Winter's avatar
Justus Winter committed
218
            use crate::openpgp::packet::signature::subpacket::SubpacketTag;
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
            let count = sigs_subpacket_tags_count[t];
            let size_count = sigs_subpacket_tags_size_count[t];
            let tag_name = format!("{:?}", SubpacketTag::from(t as u8));
            let tag_short = String::from_utf8_lossy(
                tag_name.as_bytes().chunks(30).next().unwrap());
            if size_count > 0 {
                println!("{:>30} {:>8} {:>6} {:>4} {:>4} {:>5} {:>14}",
                         tag_short,
                         count,
                         sigs_subpacket_tags_unknown[t],
                         sigs_subpacket_tags_size_min[t],
                         sigs_subpacket_tags_size_bytes[t] / size_count,
                         sigs_subpacket_tags_size_max[t],
                         sigs_subpacket_tags_size_bytes[t]);
            } else if count > 0 {
                println!("{:>30} {:>8} {:>6} {:>4} {:>4} {:>5} {:>14}",
                         tag_short,
                         count,
                         sigs_subpacket_tags_unknown[t],
                         "-", "-", "-", "-");
            }
        }
241 242
    }

243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
    if ua_invalid_count > 0
        || ua_image_count.iter().any(|c| *c > 0)
        || ua_unknown_count.iter().any(|c| *c > 0)
    {
        println!();
        println!("# User Attribute Subpacket statistics");
        println!();
        println!("{:>18} {:>9}",
                 "", "count",);
        println!("----------------------------");
        for t in 0..256 {
            let n = ua_image_count[t];
            if n > 0 {
                println!("{:>18} {:>9}",
                         match t {
                             1 =>         "Image::JPEG".into(),
                             100...110 => format!("Image::Private({})", t),
                             _ =>         format!("Image::Unknown({})", t),
                         }, n);
            }
        }
        for t in 0..256 {
            let n = ua_unknown_count[t];
            if n > 0 {
                println!("{:>18} {:>9}", format!("Unknown({})", t), n);
            }
        }
        if ua_invalid_count > 0 {
            println!("{:>18} {:>9}", "Invalid", ua_invalid_count);
        }
    }

275 276 277 278 279
    if tpk_count == 0 {
        return;
    }

    println!();
280
    println!("# TPK statistics\n\n\
281
              {:>22} {:>9} {:>9} {:>9}",
282
             "", "min", "mean", "max");
283 284
    println!("----------------------------------------------------");
    println!("{:>22} {:>9} {:>9} {:>9}",
285 286
             "Size (packets)",
             tpk_min.packets, packet_count / tpk_count, tpk_max.packets);
287
    println!("{:>22} {:>9} {:>9} {:>9}",
288 289
             "Size (bytes)",
             tpk_min.bytes, packet_size / tpk_count, tpk_max.bytes);
290 291

    println!("\n{:>22}", "- Packets -");
292 293 294
    for t in 0..64 {
        let max = tpk_max.tags[t];
        if t as u8 != Tag::PublicKey.into() && max > 0 {
295
            println!("{:>22} {:>9} {:>9} {:>9}",
296 297 298 299 300 301
                     format!("{:?}", Tag::from(t as u8)),
                     tpk_min.tags[t],
                     tags_count[t] / tpk_count,
                     max);
        }
    }
302 303 304 305 306 307 308 309 310 311 312 313 314

    println!("\n{:>22}", "- Signatures -");
    for t in 0..256 {
        let max = tpk_max.sigs[t];
        if max > 0 {
            println!("{:>22} {:>9} {:>9} {:>9}",
                     format!("{:?}",
                             SignatureType::from(t as u8)),
                     tpk_min.sigs[t],
                     sigs_count[t] / tpk_count,
                     max);
        }
    }
315 316 317 318 319 320
}

struct PerTPK {
    packets: usize,
    bytes: usize,
    tags: Vec<u32>,
321
    sigs: Vec<u32>,
322 323 324 325 326 327 328 329
}

impl PerTPK {
    fn min() -> Self {
        PerTPK {
            packets: 0,
            bytes: 0,
            tags: vec![0; 64],
330
            sigs: vec![0; 256],
331 332 333 334 335 336 337 338
        }
    }

    fn max() -> Self {
        PerTPK {
            packets: ::std::usize::MAX,
            bytes: ::std::usize::MAX,
            tags: vec![::std::u32::MAX; 64],
339
            sigs: vec![::std::u32::MAX; 256],
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
        }
    }

    fn update_min_max(&self, min: &mut PerTPK, max: &mut PerTPK) {
        if self.packets < min.packets {
            min.packets = self.packets;
        }
        if self.packets > max.packets {
            max.packets = self.packets;
        }
        if self.bytes < min.bytes {
            min.bytes = self.bytes;
        }
        if self.bytes > max.bytes {
            max.bytes = self.bytes;
        }
        for i in 0..64 {
            if self.tags[i] < min.tags[i] {
                min.tags[i] = self.tags[i];
            }
            if self.tags[i] > max.tags[i] {
                max.tags[i] = self.tags[i];
            }
        }
364 365 366 367 368 369 370 371
        for i in 0..256 {
            if self.sigs[i] < min.sigs[i] {
                min.sigs[i] = self.sigs[i];
            }
            if self.sigs[i] > max.sigs[i] {
                max.sigs[i] = self.sigs[i];
            }
        }
372 373
    }
}