Commit 347d712a authored by segfault's avatar segfault

Improve handling of config files, option initialization, and more

- Implement bind mounting config and data files
- Rework handling of config files
- Rework option initialization
- Implement autostarting services
- Improve error handling
parent 493cdfd3
from logging import getLogger
from os import path
import sh
logger = getLogger(__name__)
from logging import getLogger
import os
from os import path
import sh
from pwd import getpwnam
from grp import getgrnam
from abc import ABCMeta, abstractmethod
import configparser
from onionkit.util import open_locked
from onionkit.exceptions import FileNotEmptyError, OptionNotFoundError
logger = getLogger(__name__)
class BindMount(object):
# Path to the source of the bind mount. Relative to service's state directory if not an absolute path.
source: str
# Absolute path to the target of the bind mount
target: str
owner = "root"
group = "root"
mode: int = None
def __init__(self,
source: str = None,
target: str = None,
owner: str = None,
group: str = None,
mode: int = None,
is_dir=False):
if source is not None:
self.source = source
if target is not None:
self.target = target
if owner is not None:
self.owner = owner
if group is not None:
self.group = group
if mode is not None:
self.mode = mode
self.is_dir = is_dir
self.created_empty_target = False
if self.mode is None:
self.mode = 0o750 if self.is_dir else 0o640
def create(self):
logger.debug("Creating bind-mount %r", self.source)
if self.is_dir:
os.mkdir(self.source, mode=self.mode)
else:
os.close(os.open(self.source, os.O_CREAT, mode=self.mode))
os.chown(self.source, uid=getpwnam(self.owner).pw_uid, gid=getgrnam(self.group).gr_gid)
os.chmod(self.source, mode=self.mode)
def is_mounted(self):
try:
sh.findmnt("--mountpoint", self.target)
except sh.ErrorReturnCode_1:
return False
return True
def mount(self):
logger.debug("Bind-mounting %r to %r", self.source, self.target)
self.ensure_target_exists()
sh.mount("--bind", self.source, self.target)
def ensure_target_exists(self):
if path.exists(self.target):
return
if self.is_dir:
logger.debug("Creating target directory %r", self.target)
sh.mkdir(self.target)
else:
logger.debug("Creating target file %r", self.target)
sh.touch(self.target)
self.created_empty_target = True
def unmount(self):
logger.debug("Unmounting %r", self.target)
sh.umount(self.target)
if self.created_empty_target:
self.remove_empty_target()
def remove_empty_target(self):
if path.isdir(self.source):
logger.debug("Removing target directory %r", self.target)
sh.rmdir(self.target)
else:
logger.debug("Removing target file %r", self.target)
with open_locked(self.target) as f:
if f.read():
raise FileNotEmptyError("Refusing to remove target file %r: File not empty", self.target)
sh.rm(self.target)
class ConfigFile(BindMount, metaclass=ABCMeta):
@property
@abstractmethod
def source(self) -> str:
pass
@property
@abstractmethod
def target(self) -> str:
pass
@property
@abstractmethod
def default_content(self) -> str:
pass
mode = 0o640
def __init__(self):
super().__init__()
def create(self):
logger.debug("Creating config file %r", self.source)
with open_locked(self.source, "w") as f:
os.fchown(f.fileno(), uid=getpwnam(self.owner).pw_uid, gid=getgrnam(self.group).gr_gid)
os.fchmod(f.fileno(), mode=self.mode)
f.write(self.default_content)
# XXX: DELETE THIS
# def delete_lines_starting_with(self, s: str):
# with open_locked(self.source, 'r+') as f:
# lines = f.readlines()
# new_lines = [line for line in lines if not line.startswith(s)]
# f.seek(0)
# f.truncate()
# f.writelines(new_lines)
class IniLikeConfigFile(ConfigFile, metaclass=ABCMeta):
def __init__(self, *args, **kwargs):
super().__init__()
self.config = configparser.ConfigParser(*args, **kwargs)
def get(self, section: str, key: str) -> str:
try:
with open_locked(self.source) as f:
self.config.read_string(f.read())
return self.config.get(section, key)
except (configparser.NoSectionError, configparser.NoOptionError):
raise OptionNotFoundError("Property %r not found in config file %r" % (key, self.source))
def set(self, section: str, key: str, value: str):
with open_locked(self.source, 'r+') as f:
self.config.read_string(f.read())
if not self.config.has_section(section):
self.config.add_section(section)
self.config.set(section, key, value)
f.seek(0)
f.truncate()
self.config.write(f)
def remove(self, section: str, option: str):
with open_locked(self.source, 'r+') as f:
self.config.read_string(f.read())
self.config.remove_option(section, option)
f.seek(0)
f.truncate()
self.config.write(f)
......@@ -34,3 +34,11 @@ class OptionNotInitializedError(Exception):
def __init__(self, option=None):
msg = "Option %r accessed before it was initialized" % option if option else None
super().__init__(msg)
class OptionNotFoundError(Exception):
pass
class FileNotEmptyError(Exception):
pass
......@@ -54,32 +54,6 @@ def remove_line_if_present(file_path, line_):
return removed
def delete_lines_starting_with(file_path, s):
with open_locked(file_path, 'r') as f:
lines = f.readlines()
lines = [line for line in lines if not line.startswith(s)]
with open_locked(file_path, 'w+') as f:
f.writelines(lines)
def insert_to_section(file_path, section_name, s):
def write_to_file():
with open_locked(file_path, 'w+') as f:
f.writelines(lines)
with open_locked(file_path, 'r') as f:
lines = f.readlines()
for i, line in enumerate(lines):
if line.startswith("[%s]" % section_name):
lines.insert(i+1, s)
write_to_file()
return
lines.append("[%s]" % section_name)
lines.append(s)
write_to_file()
def delete_section(file_path, section_name):
def get_lines_of_section(lines, offset):
i = 0
......
......@@ -5,6 +5,7 @@ from os import path
from onionkit.file_util import open_locked
from onionkit.dbus.object import DBusObject
from onionkit.exceptions import OptionNotFoundError
# Only required for type hints
from typing import TYPE_CHECKING, Type
......@@ -18,10 +19,6 @@ CONFIG_DIR_PREFIX = "config_"
logger = getLogger(__name__)
class OptionNotFoundError(Exception):
pass
class OnionServiceOption(DBusObject, metaclass=abc.ABCMeta):
dbus_info = '''
<node>
......@@ -48,8 +45,8 @@ class OnionServiceOption(DBusObject, metaclass=abc.ABCMeta):
super().__init__()
logger.debug("Initializing option %r", self.Name)
self.service = service
self._value = self.Default
self.Reload()
self.register()
# ----- Exported functions ----- #
......@@ -130,21 +127,17 @@ class OnionServiceOption(DBusObject, metaclass=abc.ABCMeta):
"""The option's type. This is used in load() to convert the value read from file."""
pass
# whether the option will be immediately registered via DBus or only after the service was installed
register_immediately = False
# the option value will be reloaded in the GUI after the service was successfully started (
# i.e. systemd unit started, the onion service started, and the descriptor was uploaded).
# This is useful for options which change when the service starts.
# XXX: Handle this in onionkit instead of the GUI
# the option value will be reloaded in the GUI after the service was successfully started
# (i.e. systemd unit started, the onion service started, and the descriptor was uploaded).
# This is useful for options which change when the service starts, e.g. Mumble's TLS fingerprint.
reload_after_service_started = False
# ----- Not exported functions ----- #
def load(self):
"""Load the option's value from the option file. Create the option file if it doesn't exist.
If this option has any other stable representation (e.g. in a config file), this function
should be overwritten and load this other representation instead"""
If the value of this option is stored anywhere else (e.g. in a config file), this function
should be overwritten and load the value from this other place instead"""
logger.debug("Loading option %r", self.Name)
if not path.isfile(self.service.options_file):
......
......@@ -8,4 +8,3 @@ class Autostart(OnionServiceOption):
Group = "generic-checkbox"
Default = False
type = bool
register_immediately = True
\ No newline at end of file
......@@ -12,7 +12,6 @@ class Persistence(OnionServiceOption):
Group = "generic-checkbox"
Default = False
type = bool
register_immediately = True
def apply(self):
super().apply()
......@@ -37,12 +36,12 @@ class Persistence(OnionServiceOption):
except (sh.ErrorReturnCode_1, FileExistsError):
logging.error("Error while moving persistent files", exc_info=True)
self.service.mount_persistent_files()
self.service.mount_files()
def remove_persistence(self):
logging.info("Removing persistence of %r", self.service.Name)
try:
self.service.unmount_persistent_files()
self.service.unmount_files()
except sh.ErrorReturnCode_32:
logging.error("Error while unmounting persistent files", exc_info=True)
......
......@@ -2,7 +2,6 @@ import os
from os import path
import shutil
import abc
import sh
from colorlog import getLogger
from contextlib import contextmanager
......@@ -12,7 +11,6 @@ import yaml.resolver
from onionkit import _
from onionkit import crypto_util
from onionkit import util
from onionkit.file_util import open_locked
from onionkit.dbus.object import DBusObject
from onionkit.service_status import Status
......@@ -39,7 +37,7 @@ from onionkit.systemd import SystemdManager
from onionkit.tor import TorManager, HSException
# Only required for type hints
from typing import TYPE_CHECKING, List, Union, Dict, Any
from typing import TYPE_CHECKING, List, Dict
if TYPE_CHECKING:
from onionkit.option import OnionServiceOption
......@@ -116,8 +114,6 @@ class OnionService(DBusObject, metaclass=abc.ABCMeta):
self.options_file = path.join(self.state_dir, "options")
self._virtual_port = self.default_virtual_port
self.create_state_dir()
# ----- Exported functions ----- #
def Install(self):
......@@ -127,8 +123,22 @@ class OnionService(DBusObject, metaclass=abc.ABCMeta):
with self.prevent_service_autostart():
PackageKitManager(self).install(self.packages)
self.TransactionStatus = "create directories"
self.configure()
self.TransactionStatus = "service configuration"
self.create_state_dir()
self.prepare_bind_mounts()
self.pre_option_initialization_hook()
self.TransactionStatus = "bind-mounting config files"
self.bind_mount()
self.TransactionStatus = "initializing options"
self.initialize_options()
self.TransactionStatus = "more service configuration"
self.post_option_initialization_hook()
self.TransactionStatus = "creating onion address"
self.create_onion_address()
self.Status = Status.STOPPED
self.update_installation_status()
......@@ -138,22 +148,32 @@ class OnionService(DBusObject, metaclass=abc.ABCMeta):
raise
def Uninstall(self):
logger.info("Uninstalling service %r", self.Name)
self.Status = Status.UNINSTALLING
try:
logger.info("Uninstalling service %r", self.Name)
PackageKitManager(self).uninstall(self.packages)
self.remove_options_file()
self.remove_onion_address()
except Exception as e:
logger.exception(e)
self.Status = Status.ERROR
try:
self.TransactionStatus = "unmount config files"
self.unmount()
except Exception as e:
logger.exception(e)
self.Status = Status.ERROR
try:
self.TransactionStatus = "remove configuration"
self.remove_configuration()
except Exception as e:
logger.exception(e)
self.Status = Status.ERROR
if not self.IsInstalled:
self.Status = Status.NOT_INSTALLED
self.update_installation_status()
except:
logger.warning("Handling error in service.Uninstall")
if self.IsInstalled:
self.Status = Status.ERROR
else:
self.Status = Status.NOT_INSTALLED
self.update_installation_status()
raise
def Start(self):
logger.info("Starting service %r", self.Name)
......@@ -382,14 +402,14 @@ class OnionService(DBusObject, metaclass=abc.ABCMeta):
def virtual_port(self):
"""The port exposed via the onion service, i.e. the port clients have to use
to connect to the service"""
if VirtualPort.Name in self.options_dict:
return self.options_dict[VirtualPort.Name].Value
if VirtualPort in self.option_classes:
return self.options_dict[VirtualPort.__name__].Value
else:
return self.default_virtual_port
persistent_paths = list()
bind_mounts = list()
options = [
option_classes = [
VirtualPort,
Persistence,
Autostart,
......@@ -397,22 +417,7 @@ class OnionService(DBusObject, metaclass=abc.ABCMeta):
AllowLAN,
]
_options_dict = None
@property
def options_dict(self) -> Dict[str, "OnionServiceOption"]:
if not self._options_dict:
logger.debug("Initializing options of service %r", self.Name)
self._options_dict = dict()
for option in self.options:
# Some options depend on each other, for example AllowLAN depends on TargetPort.
# This causes TargetPort to get initialized during the initialization of AllowLAN.
# We test here whether the option is already initialized, so we don't do it twice.
if option.__name__ not in self._options_dict:
self._options_dict[option.__name__] = option(self)
else:
logger.debug("Option %r is already initialized", option.Name)
return self._options_dict
options_dict: Dict[str, "OnionServiceOption"] = dict()
# ----- Not exported functions ----- #
......@@ -421,41 +426,57 @@ class OnionService(DBusObject, metaclass=abc.ABCMeta):
if option.reload_after_service_started:
option.Reload()
def configure(self):
"""Initial configuration after installing the service"""
logger.info("Configuring service %r", self.Name)
self.create_onion_address()
def pre_option_initialization_hook(self):
"""Configuration that must be done before options are initialized"""
pass
def post_option_initialization_hook(self):
"""Configuration that must be done after options are initialized"""
pass
def remove_configuration(self):
"""Remove configuration which was created during installation. Part of uninstallation process."""
self.remove_state_dir()
# Register options which were not already registered
options_to_register = (option for option in self.options_dict.values() if not option.registered)
for option in options_to_register:
option.register()
if options_to_register:
self.emit_signal("org.freedesktop.DBus.Properties", "PropertiesChanged", {"Options": self.Options}, "a{sao}")
def initialize_options(self):
"""Initialize options which are not already initialized"""
uninitialized_options = (o for o in self.option_classes if o.__name__ not in self.options_dict)
for option in uninitialized_options:
self.options_dict[option.__name__] = option(self)
if uninitialized_options:
self.emit_signal("org.freedesktop.DBus.Properties",
"PropertiesChanged", {"Options": self.Options}, "a{sao}")
def restore(self):
# XXX: Fix me
logger.info("Restoring service %r", self.Name)
self.Install()
self.mount_persistent_files()
self.bind_mount()
for option in self.options_dict:
if option != "persistence":
if option != "Persistence":
self.options_dict[option].apply()
@staticmethod
def ensure_target_exists(persistence_record):
if path.exists(persistence_record.target_path):
return
if path.isdir(persistence_record.persistence_path):
sh.mkdir(persistence_record.target_path)
else:
sh.touch(persistence_record.target_path)
@staticmethod
def remove_target(persistence_record):
if path.isdir(persistence_record.persistence_path):
sh.rmdir(persistence_record.target_path)
else:
sh.rm(persistence_record.target_path)
def bind_mount(self):
"""Bind mount config and data files and directories"""
logger.info("Mounting bind mounts for %r", self.Name)
for bind_mount in self.bind_mounts:
bind_mount.mount()
def prepare_bind_mounts(self):
for bind_mount in self.bind_mounts:
# Expand the bind mount source paths, which are relative to the service's state directory
if not path.isabs(bind_mount.source):
bind_mount.source = path.join(self.state_dir, bind_mount.source)
# Create the source file / directory
bind_mount.create()
def unmount(self):
"""Unmount bind mounts"""
logger.info("Unmounting bind mounts for %r", self.Name)
for bind_mount in self.bind_mounts:
bind_mount.unmount()
def create_state_dir(self):
try:
......
from logging import getLogger
import threading
import fcntl
from contextlib import contextmanager
from typing import TextIO
from gi.repository import GLib
logger = getLogger(__name__)
def run_threaded(function, *args):
thread = threading.Thread(target=function, args=args)
thread.start()
......@@ -10,4 +18,19 @@ def run_threaded(function, *args):
def process_mainloop_events():
context = GLib.MainLoop().get_context()
while context.pending():
context.iteration()
\ No newline at end of file
context.iteration()
@contextmanager
def open_locked(path, *args, **kwargs) -> TextIO:
with open(path, *args, **kwargs) as f:
try:
logger.log(5, "Acquiring file lock on %r", path)
fcntl.flock(f, fcntl.LOCK_EX)
logger.log(5, "Acquired file lock on %r", path)
yield f
finally:
logger.log(5, "Releasing file lock on %r", path)
fcntl.flock(f, fcntl.LOCK_UN)
......@@ -66,7 +66,7 @@ def print_info(service):
("address", service.address),
("local-port", service.port),
("remote-port", service.virtual_port),
("persistent-paths", service.persistent_paths),
("bind-mounts", service.bind_mounts),
("options", OrderedDict([(option.Name, option.Value) for option in
service.options_dict.values()])),
]))
......
......@@ -105,10 +105,11 @@ def main():
if service.Status == Status.INVALID:
service.Stop()
for option in service.options_dict.values():
if not option.register_immediately and not service.IsInstalled:
continue
option.register()
if service.IsInstalled:
service.initialize_options()
if "Autostart" in service.options_dict and service.options_dict["Autostart"].Value:
GLib.idle_add(service.Start())
mainloop = GLib.MainLoop()
......
import os
from os import path
import random
import string
import sh
from onionkit import _
from onionkit import file_util
from onionkit import option_util
from onionkit import option