Daemon not starting if /tmp has noexec
Problem / how to reproduce
If /tmp is mounted with the noexec option, the daemon fails to start.
Mounting /tmp with options to limit execution etc is a commonly used hardening method.
For example:
tmpfs /tmp tmpfs nodev,nosuid,noexec,size=1024M 0 0
Result
After this, running gitlab-ctl restart gitlab-pages results in:
2018-04-24_18:39:30.08949 time="2018-04-24T21:39:30+03:00" level=info msg="GitLab Pages Daemon" revision=5405831 version=dev
2018-04-24_18:39:30.08958 time="2018-04-24T21:39:30+03:00" level=info msg="URL: https://gitlab.com/gitlab-org/gitlab-pages"
2018-04-24_18:39:30.08979 time="2018-04-24T21:39:30+03:00" level=info msg="running the daemon as unprivileged user" gid=998 uid=998
2018-04-24_18:39:30.09712 time="2018-04-24T21:39:30+03:00" level=error msg="start failed" error="fork/exec /gitlab-pages: permission denied"
2018-04-24_18:39:30.11765 time="2018-04-24T21:39:30+03:00" level=fatal error="fork/exec /gitlab-pages: permission denied"Solution
This has been documented under Pages daemon fails with permission denied errors
TL;DR you can specify the value of $TMPDIR in Omnibus installations
gitlab_pages['env'] = {'TMPDIR' => '<new_tmp_path>'}Potential fix
Allow specifying tmp (or chroot, is it?) directory?
Workaround
Remount tmp without the noexec before starting gitlab-pages.
Strace
epoll_ctl(4, EPOLL_CTL_ADD, 6, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1404247728, u64=139712395419312}}) = -1 EPERM (Operation not permitted)
epoll_ctl(4, EPOLL_CTL_DEL, 6, 0xc4201135cc) = -1 EPERM (Operation not permitted)
openat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/etc/resolv.conf", O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC, 0644) = 7
epoll_ctl(4, EPOLL_CTL_ADD, 7, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1404247728, u64=139712395419312}}) = -1 EPERM (Operation not permitted)
epoll_ctl(4, EPOLL_CTL_DEL, 7, 0xc420113614) = -1 EPERM (Operation not permitted)
read(6, "# Dynamic resolv.conf(5) file fo"..., 32768) = 234
write(7, "# Dynamic resolv.conf(5) file fo"..., 234) = 234
read(6, "", 32768) = 0
close(7) = 0
close(6) = 0
mount("/var/opt/gitlab/gitlab-rails/shared/pages", "/tmp/gitlab-pages-1524595257054546934/pages", 0xc420012590, MS_BIND|MS_REC, NULL) = 0
pipe2([6, 7], O_CLOEXEC) = 0
epoll_ctl(4, EPOLL_CTL_ADD, 6, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1404247728, u64=139712395419312}}) = 0
fcntl(6, F_GETFL) = 0 (flags O_RDONLY)
fcntl(6, F_SETFL, O_RDONLY|O_NONBLOCK) = 0
epoll_ctl(4, EPOLL_CTL_ADD, 7, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1404247536, u64=139712395419120}}) = 0
fcntl(7, F_GETFL) = 0x1 (flags O_WRONLY)
fcntl(7, F_SETFL, O_WRONLY|O_NONBLOCK) = 0
fcntl(6, F_GETFL) = 0x800 (flags O_RDONLY|O_NONBLOCK)
fcntl(6, F_SETFL, O_RDONLY) = 0
pipe2([8, 9], O_CLOEXEC) = 0
getpid() = 2227
rt_sigprocmask(SIG_SETMASK, NULL, [], 8) = 0
rt_sigprocmask(SIG_SETMASK, ~[], NULL, 8) = 0
clone(child_stack=0, flags=CLONE_VM|CLONE_VFORK|SIGCHLD) = 2234
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
close(9) = 0
read(8, "\r\0\0\0\0\0\0\0", 8) = 8
close(8) = 0
wait4(2234, [{WIFEXITED(s) && WEXITSTATUS(s) == 253}], 0, NULL) = 2234
write(2, "\33[31mERRO\33[0m[0000] start failed"..., 126ERRO[0000] start failed error="fork/exec /gitlab-pages: permission denied"
) = 126
epoll_ctl(4, EPOLL_CTL_DEL, 7, 0xc42011389c) = 0
close(7) = 0
umount2("/tmp/gitlab-pages-1524595257054546934/pages", MNT_DETACH) = 0
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934", 0) = -1 EISDIR (Is a directory)
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934", AT_REMOVEDIR) = -1 ENOTEMPTY (Directory not empty)
lstat("/tmp/gitlab-pages-1524595257054546934", {st_mode=S_IFDIR|0755, st_size=100, ...}) = 0
openat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934", O_RDONLY|O_CLOEXEC) = 7
epoll_ctl(4, EPOLL_CTL_ADD, 7, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1404247536, u64=139712395419120}}) = -1 EPERM (Operation not permitted)
epoll_ctl(4, EPOLL_CTL_DEL, 7, 0xc4201136f4) = -1 EPERM (Operation not permitted)
getdents64(7, /* 5 entries */, 4096) = 136
getdents64(7, /* 0 entries */, 4096) = 0
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/gitlab-pages", 0) = 0
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/pages", 0) = -1 EISDIR (Is a directory)
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/pages", AT_REMOVEDIR) = 0
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/etc", 0) = -1 EISDIR (Is a directory)
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/etc", AT_REMOVEDIR) = -1 ENOTEMPTY (Directory not empty)
lstat("/tmp/gitlab-pages-1524595257054546934/etc", {st_mode=S_IFDIR|0755, st_size=60, ...}) = 0
openat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/etc", O_RDONLY|O_CLOEXEC) = 8
epoll_ctl(4, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1404247536, u64=139712395419120}}) = -1 EPERM (Operation not permitted)
epoll_ctl(4, EPOLL_CTL_DEL, 8, 0xc420113634) = -1 EPERM (Operation not permitted)
getdents64(8, /* 3 entries */, 4096) = 80
getdents64(8, /* 0 entries */, 4096) = 0
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/etc/resolv.conf", 0) = 0
getdents64(8, /* 0 entries */, 4096) = 0
close(8) = 0
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/etc", 0) = -1 EISDIR (Is a directory)
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934/etc", AT_REMOVEDIR) = 0
getdents64(7, /* 0 entries */, 4096) = 0
close(7) = 0
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934", 0) = -1 EISDIR (Is a directory)
unlinkat(AT_FDCWD, "/tmp/gitlab-pages-1524595257054546934", AT_REMOVEDIR) = 0
epoll_ctl(4, EPOLL_CTL_DEL, 6, 0xc42011385c) = 0
close(6) = 0
close(5) = 0
write(2, "\33[31mFATA\33[0m[0000] "..., 126FATA[0000] error="fork/exec /gitlab-pages: permission denied"
) = 126
exit_group(1) = ?
+++ exited with 1 +++