Commit 7513514d authored by Heinz N. Gies's avatar Heinz N. Gies

Support for import of both bzip2 and gzip compressed images

parent 3a89dd84
......@@ -7,6 +7,7 @@ dependencies = [
"chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.26.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clippy 0.0.152 (registry+https://github.com/rust-lang/crates.io-index)",
"flate2 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
......@@ -20,6 +21,7 @@ dependencies = [
"slog-bunyan 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slog-scope 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slog-term 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
......@@ -265,6 +267,15 @@ name = "encode_unicode"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "flate2"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
"miniz-sys 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "foreign-types"
version = "0.2.0"
......@@ -458,6 +469,15 @@ dependencies = [
"unicase 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "miniz-sys"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "mio"
version = "0.6.10"
......@@ -919,6 +939,17 @@ dependencies = [
"rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempfile"
version = "2.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "term"
version = "0.4.6"
......@@ -1174,6 +1205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90"
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
"checksum encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c088ec0ed2282dcd054f2c124c0327f953563e6c75fdc6ff5141779596289830"
"checksum flate2 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)" = "36df0166e856739905cd3d7e0b210fe818592211a008862599845e012d8d304c"
"checksum foreign-types 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e4056b9bd47f8ac5ba12be771f77a0dae796d1bbaaf5fd0b9c2d38b69b8a29d"
"checksum futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4b63a4792d4f8f686defe3b39b92127fea6344de5d38202b2ee5a11bbbf29d6a"
"checksum futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a283c84501e92cade5ea673a2a7ca44f71f209ccdd302a3e0896f50083d2c5ff"
......@@ -1199,6 +1231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376"
"checksum memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1dbccc0e46f1ea47b9f17e6d67c5a96bd27030519c519c9c91327e31275a47b4"
"checksum mime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "153f98dde2b135dece079e5478ee400ae1bab13afa52d66590eacfc40e912435"
"checksum miniz-sys 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "28eaee17666671fa872e567547e8428e83308ebe5808cdf6a0e28397dbe2c726"
"checksum mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "dbd91d3bfbceb13897065e97b2ef177a09a438cb33612b2d371bf568819a9313"
"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
"checksum native-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04b781c9134a954c84f0594b9ab3f5606abc516030388e8511887ef4c204a1e5"
......@@ -1252,6 +1285,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5"
"checksum take_mut 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7986ceb18a0d75e1fcb8b27c0119389bbe05f016e5a6e54d003251acc1122108"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"
"checksum tempfile 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "5b92290d7f1ce2d221405d5c78b9c568c9f1debb314aa92a513cd99db709f931"
"checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1"
"checksum term_size 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2b6b55df3198cc93372e85dd2ed817f0e38ce8cc0f22eb32391bfad9c4bf209"
"checksum textwrap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f728584ea33b0ad19318e20557cb0a39097751dbb07171419673502f848c7af6"
......
......@@ -22,7 +22,9 @@ regex = "0.2"
rand = "0.3"
reqwest = "0.7"
chrono = { version = "0.4", features = ["serde", "rustc-serialize"] }
bzip2 = "0.3.2"
bzip2 = "0.3"
flate2 = "0.2"
tempfile = "2.1"
# indicatif = "0.5"
[dependencies.clap]
......
......@@ -102,8 +102,15 @@ subcommands:
subcommands:
- avail:
about: lists available images
- show:
about: show manifest of an available image
args:
- uuid:
help: UUID if the image to get
index: 1
required: true
- get:
about: lists available images
about: info on an installed image
args:
- uuid:
help: UUID if the image to get
......
......@@ -19,6 +19,8 @@ pub struct Settings {
pub repo: String,
#[serde(default = "default_conf_dir")]
pub conf_dir: String,
#[serde(default = "default_image_dir")]
pub image_dir: String,
#[serde(default = "devfs_ruleset")]
pub devfs_ruleset: u32,
pub networks: Map<String, String>,
......@@ -38,6 +40,10 @@ fn default_conf_dir() -> String {
"/etc/jails".to_string()
}
fn default_image_dir() -> String {
"/var/imgadm/images".to_string()
}
fn default_repo() -> String {
"https://datasets.project-fifo.net/images".to_string()
}
......
use std::io::Read;
use std::io::{Read, Seek, SeekFrom};
use std::error::Error;
use std::fs::File;
use std::io::copy;
use config::Config;
use errors::GenericError;
use zfs;
use reqwest;
use tempfile;
use serde_json;
use uuid::Uuid;
use chrono::{DateTime, Utc};
......@@ -13,7 +16,8 @@ use prettytable::Table;
use prettytable::format;
use prettytable::row::Row;
use prettytable::cell::Cell;
use bzip2::read::BzDecoder;
use flate2::read::GzDecoder;
#[derive(Debug, Serialize, Deserialize, Clone)]
......@@ -131,24 +135,80 @@ pub fn get(config: &Config, uuid: Uuid) -> Result<i32, Box<Error>> {
Ok(0)
}
pub fn show(config: &Config, uuid: Uuid) -> Result<i32, Box<Error>> {
let mut url = config.settings.repo.clone();
let uuid_str = uuid.hyphenated().to_string();
url.push('/');
url.push_str(uuid_str.as_str());
debug!("Fethcing image"; "repo" => config.settings.repo.clone(),
"uuid" => uuid_str.clone(), "url" => url.clone());
let resp = reqwest::get(url.as_str())?;
let image = Image::from_reader(resp)?;
let j = serde_json::to_string_pretty(&image)?;
println!("{}\n", j);
//print_images(images, false, false);
Ok(0)
}
pub fn import(config: &Config, uuid: Uuid) -> Result<i32, Box<Error>> {
let mut url = config.settings.repo.clone();
let uuid_str = uuid.hyphenated().to_string();
let mut dataset = config.settings.pool.clone();
dataset.push('/');
dataset.push_str(uuid_str.as_str());
url.push('/');
url.push_str(uuid_str.as_str());
if zfs::is_present(dataset.as_str()) {
return Err(GenericError::bx("Dataset already present"));
};
debug!("Fethcing image"; "repo" => config.settings.repo.clone(),
"uuid" => uuid_str.clone(), "url" => url.clone());
let resp = reqwest::get(url.as_str())?;
let image = Image::from_reader(resp)?;
match image.origin {
None => (),
Some(origin) => {
let mut origin_dataset = config.settings.pool.clone();
origin_dataset.push('/');
origin_dataset.push_str(origin.hyphenated().to_string().as_str());
if ! zfs::is_present(origin_dataset.as_str()) {
import(config, origin)?;
}
}
};
let file_info = image.files[0].clone();
let mut file = uuid_str.clone();
file.push_str(".");
file.push_str(file_info.compression.as_str());
url.push_str("/file");
let mut out = File::create(file)?;
let mut out: File = tempfile::tempfile()?;
let mut resp = reqwest::get(url.as_str())?;
println!("Downloading {} ...", uuid_str.as_str());
copy(&mut resp, &mut out)?;
//print_images(images, false, false);
println!("Importing {} ...", uuid_str.as_str());
out.seek(SeekFrom::Start(0))?;
match file_info.compression.as_str() {
"bzip2" => {
let mut decompressor = BzDecoder::new(out);
zfs::receive(dataset.as_str(), &mut decompressor)?;
}
"gzip" => {
let mut decompressor = GzDecoder::new(out)?;
zfs::receive(dataset.as_str(), &mut decompressor)?;
}
compression => {
println!("Encountered {} compression", compression);
return Err(GenericError::bx("Only bzip2 compression is supporred for images."));
}
}
let mut cfg_path = config.settings.image_dir.clone();
cfg_path.push('/');
cfg_path.push_str(uuid_str.as_str());
cfg_path.push_str(".json");
println!("Writing manifest file: {}", cfg_path);
let cfg_file = File::create(cfg_path)?;
serde_json::to_writer(cfg_file, &image)?;
Ok(0)
}
......@@ -25,7 +25,9 @@ extern crate regex;
extern crate rand;
extern crate reqwest;
extern crate chrono;
extern crate tempfile;
extern crate bzip2;
extern crate flate2;
//extern crate indicatif;
......@@ -289,7 +291,7 @@ fn console(conf: &Config, matches: &clap::ArgMatches) -> Result<i32, Box<Error>>
let mut child = Command::new(JEXEC)
.args(&[jid.id.to_string().as_str(), "/bin/csh"])
.spawn()
.expect("failed to execute jsj");
.expect("failed to execute jexec");
let ecode = child.wait().expect("failed to wait on child");
if ecode.success() {
Ok(0)
......@@ -525,6 +527,7 @@ fn images(conf: &Config, matches: &clap::ArgMatches) -> Result<i32, Box<Error>>
match matches.subcommand() {
("avail", Some(avail_matches)) => images_avail(&conf, avail_matches),
("get", Some(get_matches)) => images_get(&conf, get_matches),
("show", Some(show_matches)) => images_show(&conf, show_matches),
("import", Some(import_matches)) => images_import(&conf, import_matches),
("", None) => {
Ok(0)
......@@ -543,6 +546,12 @@ fn images_get(conf: &Config, matches: &clap::ArgMatches) -> Result<i32, Box<Erro
images::get(conf, uuid)
}
fn images_show(conf: &Config, matches: &clap::ArgMatches) -> Result<i32, Box<Error>> {
let uuid_string = value_t!(matches, "uuid", String).unwrap();
let uuid = Uuid::parse_str(uuid_string.as_str()).unwrap();
images::show(conf, uuid)
}
fn images_import(conf: &Config, matches: &clap::ArgMatches) -> Result<i32, Box<Error>> {
let uuid_string = value_t!(matches, "uuid", String).unwrap();
let uuid = Uuid::parse_str(uuid_string.as_str()).unwrap();
......
......@@ -2,19 +2,21 @@
//! Wrapper around zfs commands
use std::error::Error;
use std::process::Command;
use errors::GenericError;
// #[derive(Debug)]
// /// Basic information about a ZFS dataset
// pub struct ZFSEntry {
// name: String,
// used: u64,
// avail: u64,
// refer: u64,
// mountpoint: String,
// }
use std::process::{Command, Stdio};
use errors::{GenericError, NotFoundError};
use std::io::Read;
use std::io::copy;
#[derive(Debug)]
/// Basic information about a ZFS dataset
pub struct ZFSEntry {
name: String,
used: u64,
avail: u64,
refer: u64,
mountpoint: String,
}
// /// reads the zfs datasets in a pool
// pub fn list(pool: &str) -> Result<Vec<ZFSEntry>, Box<Error>> {
......@@ -34,20 +36,39 @@ use errors::GenericError;
// Ok(res)
// }
// /// reads the zfs datasets in a pool
// pub fn get(dataset: &str) -> Result<ZFSEntry, Box<Error>> {
// debug!("Reading ZFS dataset"; "dataset" => dataset);
// let output = Command::new("zfs")
// .args(&["list", "-p", "-H", dataset])
// .output()
// .expect("zfs list failed");
// if output.status.success() {
// let reply = String::from_utf8_lossy(&output.stdout).to_string();
// deconstruct_entry(reply.as_str())
// } else {
// Err(GenericError::bx("Failed to get dataset"))
// }
// }
pub fn receive<R>(dataset: &str, mut reader: &mut R) -> Result<i32, Box<Error>>
where
R: Read,
{
let mut rec = Command::new("zfs")
.args(&["receive", dataset])
.stdin(Stdio::piped())
.spawn().unwrap();
let mut stdin = rec.stdin.take().unwrap();
copy(&mut reader, &mut stdin)?;
Ok(0)
}
/// checks weather a dataset exists or not
pub fn is_present(dataset: &str) -> bool {
get(dataset).is_ok()
}
/// reads the zfs datasets in a pool
fn get(dataset: &str) -> Result<ZFSEntry, Box<Error>> {
debug!("Reading ZFS dataset"; "dataset" => dataset);
let output = Command::new("zfs")
.args(&["list", "-p", "-H", dataset])
.output()
.expect("zfs list failed");
if output.status.success() {
let reply = String::from_utf8_lossy(&output.stdout).to_string();
deconstruct_entry(reply.as_str())
} else {
Err(GenericError::bx("Failed to get dataset"))
}
}
/// reads the zfs datasets in a pool
pub fn origin(dataset: &str) -> Result<String, Box<Error>> {
......@@ -135,32 +156,32 @@ pub fn destroy(dataset: &str) -> Result<i32, Box<Error>> {
}
// /// deconstructs a line from zfs list into an `ZFSEntry`.
// fn deconstruct_entry(line: &str) -> Result<ZFSEntry, Box<Error>> {
// let mut parts = line.split('\t');
// let name = parts.next().ok_or_else(
// || GenericError::bx("NAME field missing"),
// )?;
// let n0 = parts.next().ok_or_else(
// || GenericError::bx("USED field missing"),
// )?;
// let used: u64 = n0.parse()?;
// let n1 = parts.next().ok_or_else(
// || GenericError::bx("AVAIL field missing"),
// )?;
// let avail: u64 = n1.parse()?;
// let n2 = parts.next().ok_or_else(
// || GenericError::bx("REFER field missing"),
// )?;
// let refer: u64 = n2.parse()?;
// let mountpoint = parts.next().ok_or_else(
// || GenericError::bx("MOUNTPOINT field missing"),
// )?;
// Ok(ZFSEntry {
// name: String::from(name),
// used: used,
// avail: avail,
// refer: refer,
// mountpoint: String::from(mountpoint),
// })
// }
fn deconstruct_entry(line: &str) -> Result<ZFSEntry, Box<Error>> {
let mut parts = line.split('\t');
let name = parts.next().ok_or_else(
|| GenericError::bx("NAME field missing"),
)?;
let n0 = parts.next().ok_or_else(
|| GenericError::bx("USED field missing"),
)?;
let used: u64 = n0.parse()?;
let n1 = parts.next().ok_or_else(
|| GenericError::bx("AVAIL field missing"),
)?;
let avail: u64 = n1.parse()?;
let n2 = parts.next().ok_or_else(
|| GenericError::bx("REFER field missing"),
)?;
let refer: u64 = n2.parse()?;
let mountpoint = parts.next().ok_or_else(
|| GenericError::bx("MOUNTPOINT field missing"),
)?;
Ok(ZFSEntry {
name: String::from(name),
used: used,
avail: avail,
refer: refer,
mountpoint: String::from(mountpoint),
})
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment