Commit e44990b6 authored by MrMan's avatar MrMan

Merge branch '22-add-dovecot-and-e2e-test' into 'master'

Resolve "Add Dovecot component, fill out E2E test suite"

Closes #22

See merge request !20
parents 0dfcbfa4 2be18635
Pipeline #50086803 passed with stage
in 7 minutes and 22 seconds
......@@ -15,12 +15,6 @@ The [`postmgr` container][gitlab-docker-repo] is the primary container of this p
You **should** be able to download the single container, connect it to your container orchestration process of choice, and have a fully functional, mostly secure, manageable Postfix installation. If this is ever not-true, [file a bug][gitlab-issues].
## Binary ##
`postmgr` can be used by download the binary from a trusted source or build from source.
TODO: table listing where to download the binary
# Why postmgr? #
While some might [advise against running your own mail server][do-dont-run-your-own-mailserver], for relatively simple applications that need to send mail, I often find it's easiest to set up a small Postfix instance to handle things like registration emails and the like. While the documentation is extensive, and many resources abound, managing Postfix can be a very difficult endeavor, with misleading/overloaded terminology, and seemingly cryptic settings.
......@@ -31,11 +25,47 @@ One of the express goals of this project is to make management, configuration, a
# Architecture #
The basic architecture of `postmgr` is pretty simple: `postmgr` runs and manages Postfix as a forked process.
TODO: Architecture image
`postmgr` listens on the ports you would expect Postfix to listen on, plus a few more for value-added features. Most features on `postmgr` can be enabled/disabled. The default configuration for `postmgr` does nothing more than serve as a thin daemonized wrapper over postfix itself.
The basic architecture of `postmgr` is pretty simple:
```
+------------------------------+
| |
| Postfix Component |
| | +-----------------+
| | | |
| +----------+ +----------+ +<--------+ Dovecot |
| | | | | | | |
| | postfix | |MailboxDB | +-------->+ +-----------+ |
| | process | | | | | | | |
| | | | | | | |dovecot | |
| +----------+ +----------+ | | |process | |
| | | +-----------+ |
+----------+-------+-----------+ +-----------------+
| ^
| |
| |
| |
v |
+----------+-------+-----------+
| |
| |
| Web Admin Component |
| |
| |
+------------------------------+
```
There are three major components:
- Postfix (`src/components/postfix.rs`) which controls and manipulates a child `postfix` process
- Web Admin (`src/components/web_admin.rs`) which exposes an administration interface, sending messages to the Postfix component
- Dovecot (`src/components/dovecot.rs`) which controls and manipulates a child `dovecot` process
`postmgr` listens on the ports you would expect Postfix to listen on (25, 587, etc), plus a few more for value-added features (like Dovecot).
For more more information on various bits of the architecture:
- IMAP (via dovecot): `docs/imap.md`
# Developing/debugging `postmgr` #
......@@ -46,9 +76,11 @@ If you'd like to contribute to `postmgr`, use the following steps:
3. `make diesel-cli` to install diesel CLI if it's not already installed
4. `make dev-setup` to get set up
5. Make your changes
6. Run the test suite (`make test`) and make sure they pass (the more test you've added, the more likely your MR will succeed)
6. Run the test suite (`make test`, `make test-e2e`) and make sure they pass (the more test you've added, the more likely your MR will succeed)
7. Make a merge request against this repository
For more information on testing, see `docs/testing.md`
[postfix]: http://www.postfix.org/
[do-dont-run-your-own-mailserver]: https://www.digitalocean.com/community/tutorials/why-you-may-not-want-to-run-your-own-mail-server
[mailu]: https://github.com/Mailu/Mailu/
......
# IMAP #
## Supported clients ##
Curently [`dovecot`](https://wiki.dovecot.org) is the only supported IMAP client.
The `Dovecot` component (`src/components/dovecot.rs`) manages a child `dovecot` process (which is started in non-daemonized mode for better control), generating and placing configuration as necessary.
## Suported SMTP authentication method ##
Currently the following authentication methods are supported via `dovecot`:
- `AUTH PLAIN`
# Testing #
## Debugging E2E tests ##
To run the E2E tests without output suppression:
```
$ cargo test -- --ignored --nocapture
```
......@@ -104,6 +104,19 @@ session_cache_db = "btree:/var/lib/postfix/smtpd_tls_session_cache.db"
log_level = 1
session_cache_timeout_seconds = 3600
###########
# Dovecot #
###########
[dovecot]
bin_path = "/usr/sbin/dovecot"
doveadm_bin_path = "/usr/bin/doveadm"
config_output_dir = "/etc/dovecot"
mail_location = "/var/mail"
unix_socket_path = "/var/spool/postfix/private/auth"
#############
# Web Admin #
#############
......
......@@ -101,6 +101,19 @@ session_cache_db = "btree:/var/lib/postfix/smtpd_tls_session_cache.db"
log_level = 1
session_cache_timeout_seconds = 3600
###########
# Dovecot #
###########
[dovecot]
bin_path = "/usr/sbin/dovecot"
doveadm_bin_path = "/usr/bin/doveadm"
config_output_dir = "/etc/dovecot"
mail_location = "/var/mail"
unix_socket_path = "/var/spool/postfix/private/auth"
#############
# Web Admin #
#############
......
......@@ -6,6 +6,11 @@ COPY . .
# Install rsyslog configuration for postfix
COPY infra/docker/syslog/postfix-syslog.conf /etc/rsyslog.d/postfix.conf
# Add groups & users for use by postfix and/or dovecot
RUN groupadd -g 3000 submission
RUN groupadd -g 5000 vmail
RUN useradd -m -d /var/vmail -s /bin/false -u 5000 -g vmail vmail
# Set up entrypoint
COPY infra/docker/entrypoint.sh /bin/entrypoint.sh
RUN chmod =x /bin/entrypoint.sh
......
......@@ -5,11 +5,8 @@ COPY . .
# Install postfix
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install pkg-config libssl-dev ca-certificates make \
rsyslog telnet postfix \
rsyslog telnet sqlite postfix \
dovecot-imapd dovecot-pop3d dovecot-sqlite dovecot-pgsql
RUN ln -s /usr/sbin/postfix /usr/bin/postfix
# Add submission group for postfix to use
RUN groupadd -g 3000 submission
RUN cargo install
pub mod sqlite;
use components::Error;
use config::{DBCfg, DBType};
use config::{DBCfg, DBType, DovecotDBSettings};
use models::user::MailboxUser;
use models::{ModelWithUUID, PaginatedList, PaginationOptions};
use self::sqlite::SQLiteDB;
use std::path::Path;
use std::any::Any;
use rusqlite::Connection as SQLiteConnection;
pub enum DB {
......@@ -77,7 +76,7 @@ pub fn make_admin_db(db_cfg: DBCfg) -> Result<Box<AdminDB>, Error> {
pub trait MailboxDB
where
Self: Connectable + SupportsVAliasLookup + SupportsVMailboxLookup + SupportsDovecotAuth
Self: Connectable + SupportsVAliasLookup + SupportsVMailboxLookup
{
// Initialize the database connection (if necessary), idempotently performing
// all early/pre-work data changes and migrations necessary for a Postfix data holding db
......@@ -204,56 +203,36 @@ impl SupportsVMailboxLookup for DB {
}
// TODO: this can probably be generalized in the future to SupportsAuth<T> when Dovecot auth is added
pub trait SupportsDovecotAuth {
type ConfigTemplate;
type SQLConfigTemplate;
pub trait SupportsDovecotAuth where Self: MailboxDB {
/// Dovecot userdb settings
fn dovecot_userdb_settings(&self) -> Result<DovecotDBSettings, Error>;
fn dovecot_sasl_path(&self) -> Result<&'static str, Error>;
/// Dovecot passdb settings
fn dovecot_passdb_settings(&self) -> Result<DovecotDBSettings, Error>;
fn dovecot_config_dir(&self) -> Result<&'static str, Error>;
fn dovecot_config_filename(&self) -> Result<&'static str, Error>;
fn dovecot_sql_config_filename(&self) -> Result<&'static str, Error>;
fn write_dovecot_config_files(&self) -> Result<(), Error>;
/// Get the driver for dovecot
fn dovecot_driver_type(&self) -> Result<DBType, Error>;
}
impl SupportsDovecotAuth for DB {
type ConfigTemplate = Box<Any>;
type SQLConfigTemplate = Box<Any>;
fn dovecot_sasl_path(&self) -> Result<&'static str, Error> {
match self {
DB::SQLite(db) => db.dovecot_sasl_path(),
_ => Err(Error::NotSupported)
}
}
fn dovecot_config_dir(&self) -> Result<&'static str, Error> {
fn dovecot_userdb_settings(&self) -> Result<DovecotDBSettings, Error> {
match self {
DB::SQLite(db) => db.dovecot_config_dir(),
_ => Err(Error::NotSupported)
}
}
fn dovecot_config_filename(&self) -> Result<&'static str, Error> {
match self {
DB::SQLite(db) => db.dovecot_config_filename(),
_ => Err(Error::NotSupported)
DB::SQLite(db) => db.dovecot_userdb_settings(),
_ => Err(Error::NotSupported),
}
}
fn dovecot_sql_config_filename(&self) -> Result<&'static str, Error> {
fn dovecot_passdb_settings(&self) -> Result<DovecotDBSettings, Error> {
match self {
DB::SQLite(db) => db.dovecot_config_filename(),
_ => Err(Error::NotSupported)
DB::SQLite(db) => db.dovecot_passdb_settings(),
_ => Err(Error::NotSupported),
}
}
fn write_dovecot_config_files(&self) -> Result<(), Error> {
fn dovecot_driver_type(&self) -> Result<DBType, Error> {
match self {
DB::SQLite(db) => db.write_dovecot_config_files(),
_ => Err(Error::NotSupported)
DB::SQLite(db) => db.dovecot_driver_type(),
_ => Err(Error::NotSupported),
}
}
}
pub mod schema;
pub mod models;
use std::ffi::OsStr;
use std::fs::{create_dir_all, File};
use std::io::Write;
use std::path::Path;
use askama::Template;
use chrono::prelude::Local;
use rusqlite::Connection as SqliteConnection;
use rusqlite::Statement;
use self::schema::{ADMIN_MIGRATIONS, MAILBOX_MIGRATIONS};
use components::Error;
use components::db::*;
use config::SQLiteDBCfg;
use models::user::MailboxUser;
use models::{ModelWithUUID, PaginatedList, PaginationOptions, DBEntity};
use rusqlite::Connection as SqliteConnection;
use rusqlite::Statement;
use self::schema::{ADMIN_MIGRATIONS, MAILBOX_MIGRATIONS};
use std::ffi::OsStr;
use std::fs::{create_dir_all, File};
use std::io::Write;
use std::path::{Path, PathBuf};
use make_absolute_path_from_str;
pub struct SQLiteDB {
......@@ -122,7 +124,7 @@ impl MailboxDB for SQLiteDB {
Ok(())
}
fn add_mailbox_user(&self, u: MailboxUser) -> Result<ModelWithUUID<MailboxUser>, Error> {
fn add_mailbox_user(&self, mut u: MailboxUser) -> Result<ModelWithUUID<MailboxUser>, Error> {
let db_model = ModelWithUUID::from_model(u);
db_model.insert(self)?;
Ok(db_model)
......@@ -278,86 +280,43 @@ impl SupportsVMailboxLookup for SQLiteDB {
}
}
#[derive(Template)]
#[template(path = "config/dovecot/dovecot.conf")]
pub struct DovecotConfTemplate {
filename: String,
generation_time: String,
sql_config_file_abs_path: String,
abs_db_path: String
}
#[derive(Template)]
#[template(path = "config/dovecot/dovecot-sql.conf.ext")]
pub struct DovecotSQLConfTemplate {
filename: String,
generation_time: String,
abs_db_path: String
}
impl SupportsDovecotAuth for SQLiteDB {
type ConfigTemplate = DovecotConfTemplate;
type SQLConfigTemplate = DovecotSQLConfTemplate;
fn dovecot_sasl_path(&self) -> Result<&'static str, Error> { Ok("private/auth") }
fn dovecot_config_dir(&self) -> Result<&'static str, Error> { Ok("/etc/dovecot") }
fn dovecot_config_filename(&self) -> Result<&'static str, Error> { Ok("dovecot.conf") }
fn dovecot_sql_config_filename(&self) -> Result<&'static str, Error> { Ok("dovecot-sql.conf.ext") }
fn write_dovecot_config_files(&self) -> Result<(), Error> {
// Create the dovecot config directory if it doesn't exist
let config_dir = PathBuf::from(self.dovecot_config_dir()?);
if !config_dir.exists() { create_dir_all(&config_dir)?; }
let config_filename = self.dovecot_config_filename()?;
let sql_config_filename = self.dovecot_sql_config_filename()?;
// Build path & string to represent dovecot config file path
let config_file_abs_path = PathBuf::from(&config_dir).join(config_filename);
let config_file_abs_path_str = config_file_abs_path
.to_str()
.map(String::from)
.ok_or(Error::InvalidEnvironment("invalid dovecot config file path"))?;
// Build path & string to represent dovecto SQL config file path
let sql_config_file_abs_path = PathBuf::from(&config_dir).join(sql_config_filename);
let sql_config_file_abs_path_str = sql_config_file_abs_path
.to_str()
.map(String::from)
.ok_or(Error::InvalidEnvironment("invalid dovecot sql config file path"))?;
if !config_file_abs_path.is_absolute() { warn!("dovecot config file path is not absolute"); }
if !sql_config_file_abs_path.is_absolute() { warn!("dovecot sql config file path is not absolute"); }
// FIXME: mailbox_users table name is hardcoded here
const DEFAULT_USERDB_QUERY: &'static str = "SELECT home \
FROM mailbox_users \
WHERE username = '%n' AND domain = '%d'
";
// Build the template for the main config file
let template = DovecotConfTemplate {
filename: config_filename.to_string(),
generation_time: Local::now().to_string(),
sql_config_file_abs_path: sql_config_file_abs_path_str.clone(),
abs_db_path: self.make_absolute_db_path()?
};
// FIXME: mailbox_users table name is hardcoded here
const DEFAULT_PASSDB_QUERY: &'static str = "SELECT username, domain, password \
FROM mailbox_users \
WHERE username = '%n' AND domain = '%d'
";
// Write the main config file to disk
debug!("writing dovecot config file @ [{}]", config_file_abs_path_str);
let mut config_file = File::create(config_file_abs_path)?;
config_file.write_all(template.render()?.as_bytes())?;
// FIXME: mailbox_users table is hardcoded here
const DEFAULT_ITERATE_QUERY: &'static str = "SELECT username, domain FROM mailbox_users";
// Build the template for the SQLite config file
let template = DovecotSQLConfTemplate {
filename: config_filename.to_string(),
generation_time: Local::now().to_string(),
abs_db_path: self.make_absolute_db_path()?
};
// Write the main config file to disk
debug!("writing dovecot sql config file @ [{}]", sql_config_file_abs_path_str);
let mut config_file = File::create(sql_config_file_abs_path)?;
config_file.write_all(template.render()?.as_bytes())?;
impl SupportsDovecotAuth for SQLiteDB {
fn dovecot_userdb_settings(&self) -> Result<DovecotDBSettings, Error> {
// Generate the DB settings for dovecot based on this backend
Ok(DovecotDBSettings {
driver: self.dovecot_driver_type()?,
connect: self.cfg.path.clone(),
query: String::from(DEFAULT_USERDB_QUERY),
iterate_query: String::from(DEFAULT_ITERATE_QUERY),
})
}
Ok(())
fn dovecot_passdb_settings(&self) -> Result<DovecotDBSettings, Error> {
// Generate the DB settings for dovecot based on this backend
Ok(DovecotDBSettings {
driver: self.dovecot_driver_type()?,
connect: self.cfg.path.clone(),
query: String::from(DEFAULT_PASSDB_QUERY),
iterate_query: String::from(DEFAULT_ITERATE_QUERY),
})
}
fn dovecot_driver_type(&self) -> Result<DBType, Error> { Ok(DBType::SQLite) }
}
#[cfg(test)]
......
......@@ -6,7 +6,7 @@ use rusqlite::Row;
use rusqlite::types::ToSql;
use super::super::*;
const COLUMNS: &'static [&'static str] = &["uuid", "username", "domain", "password", "quota_gb"];
const COLUMNS: &'static [&'static str] = &["uuid", "username", "domain", "home", "password", "quota_gb"];
impl HasSQLTable<SQLiteDB> for ModelWithUUID<MailboxUser> {
fn sql_tbl_name() -> &'static str { "mailbox_users" }
......@@ -18,9 +18,10 @@ impl<'a, 'stmt> DBEntity<SQLiteDB, Row<'a, 'stmt>> for ModelWithUUID<MailboxUser
let uuid = row.get_checked("uuid")?;
let username = row.get_checked("username")?;
let domain = row.get_checked("domain")?;
let home = row.get_checked("home")?;
let password = row.get_checked("password")?;
let quota_gb = row.get_checked("quota_gb")?;
let model = MailboxUser { username, domain, password, quota_gb };
let model = MailboxUser { username, domain, password, home, quota_gb };
Ok(ModelWithUUID { uuid, model })
}
......@@ -32,6 +33,7 @@ impl<'a, 'stmt> DBEntity<SQLiteDB, Row<'a, 'stmt>> for ModelWithUUID<MailboxUser
&self.uuid as &ToSql,
&self.model.username,
&self.model.domain,
&self.model.home,
&self.model.password,
&self.model.quota_gb
])?;
......@@ -104,24 +106,40 @@ mod tests {
use models::user::MailboxUser;
use models::{DBEntity, ModelWithUUID, PaginationOptions};
const TEST_USER_HOME: &'static str = "/var/mail/test";
const TEST_USER_EMAIL: &'static str = "[email protected]";
const TEST_USER_DOMAIN: &'static str = "localhost";
const TEST_USER_PASSWORD: &'static str = "test";
const ADMIN_USER_HOME: &'static str = "/var/mail/admin";
const ADMIN_USER_EMAIL: &'static str = "[email protected]";
const ADMIN_USER_DOMAIN: &'static str = "localhost";
const ADMIN_USER_PASSWORD: &'static str = "admin";
fn make_test_user() -> MailboxUser {
MailboxUser::new(
String::from(TEST_USER_HOME),
String::from(TEST_USER_DOMAIN),
String::from(TEST_USER_EMAIL),
String::from(TEST_USER_PASSWORD)
)
}
fn make_admin_user() -> MailboxUser {
MailboxUser::new(
String::from(ADMIN_USER_DOMAIN),
String::from(ADMIN_USER_HOME),
String::from(ADMIN_USER_EMAIL),
String::from(ADMIN_USER_PASSWORD),
)
}
#[test]
fn models_user_insert() {
let mut db = SQLiteDB::new(SQLiteDBCfg::in_memory());
let _ = db.connect();
let model = MailboxUser::new(
String::from(TEST_USER_EMAIL),
String::from(TEST_USER_DOMAIN),
String::from(TEST_USER_PASSWORD)
);
let model = make_test_user();
let entity = ModelWithUUID::from_model(model);
let create_result = entity.insert(&db);
......@@ -133,11 +151,7 @@ mod tests {
let mut db = SQLiteDB::new(SQLiteDBCfg::in_memory());
let _ = db.connect();
let model = MailboxUser::new(
String::from(TEST_USER_EMAIL),
String::from(TEST_USER_DOMAIN),
String::from(TEST_USER_PASSWORD)
);
let model = make_test_user();
let entity = ModelWithUUID::from_model(model);
let create_result = entity.insert(&db);
......@@ -156,11 +170,7 @@ mod tests {
let mut db = SQLiteDB::new(SQLiteDBCfg::in_memory());
let _ = db.connect();
let model = MailboxUser::new(
String::from(TEST_USER_EMAIL),
String::from(TEST_USER_DOMAIN),
String::from(TEST_USER_PASSWORD)
);
let model = make_test_user();
let entity = ModelWithUUID::from_model(model);
// create the user
......@@ -182,11 +192,7 @@ mod tests {
let mut db = SQLiteDB::new(SQLiteDBCfg::in_memory());
let _ = db.connect();
let model = MailboxUser::new(
String::from(TEST_USER_EMAIL),
String::from(TEST_USER_DOMAIN),
String::from(TEST_USER_PASSWORD)
);
let model = make_test_user();
let entity = ModelWithUUID::from_model(model);
// create the user
......@@ -209,19 +215,11 @@ mod tests {
let mut db = SQLiteDB::new(SQLiteDBCfg::in_memory());
let _ = db.connect();
let test_model = MailboxUser::new(
String::from(TEST_USER_EMAIL),
String::from(TEST_USER_DOMAIN),
String::from(TEST_USER_PASSWORD)
);
let test_model = make_test_user();
let test_entity = ModelWithUUID::from_model(test_model);
let admin_model = MailboxUser::new(
String::from(ADMIN_USER_EMAIL),
String::from(ADMIN_USER_DOMAIN),
String::from(ADMIN_USER_PASSWORD),
);
let admin_model = make_admin_user();
let admin_entity = ModelWithUUID::from_model(admin_model);
// create the user
......@@ -246,11 +244,7 @@ mod tests {
let _ = db.connect();
// create the user
let model = MailboxUser::new(
String::from(TEST_USER_EMAIL),
String::from(TEST_USER_DOMAIN),
String::from(TEST_USER_PASSWORD)
);
let model = make_test_user();
let entity = ModelWithUUID::from_model(model);
let _ = entity.insert(&db).expect("test entity create failed");
......
......@@ -14,6 +14,7 @@ CREATE TABLE mailbox_users (
uuid TEXT PRIMARY KEY,
username TEXT NOT NULL,
domain TEXT NOT NULL,
home TEXT NOT NULL,
password TEXT NOT NULL,
quota_gb INTEGER DEFAULT 0,
......
use std::fs::{File, copy};
use std::io::Write;
use std::path::Path;
use std::process::{Command, Stdio, Child as ChildProcess};
use std::sync::mpsc::{sync_channel, SyncSender, Receiver};
use askama::Template;
use chrono::prelude::Local;
use make_absolute_path;
use ensure_directory;
use simple_signal::{Signal};
use simple_signal;
use components::*;
use config::{DovecotCfg, DovecotDBSettings, DovecotSettingName};
const DOVECOT_CONF_FILENAME: &'static str = "dovecot.conf";
const DOVECOT_USERDB_CONF_FILENAME: &'static str = "dovecot-sql-userdb.conf.ext";