Gitlab-workhorse has stuck subprocesses
Gitlab-workhorse ran out of files after a security scan. It had man git and gzip/bzip2 processes hanging around.
You can easily reproduce the issue by downloading a big archive and interrupting the download.
wget https://stage.code.siemens.com/ccp/linux/repository/archive.tar.bz2?ref=master
^C
Make sure there is not already a cached archive. Otherwise erase the cache. The problem is the compressor bzip2 is started but does not get the same process group as the git process. The interrupt does not kill bzip2. Here is a patch to explicitly kill the compressor:
diff --git a/archive.go b/archive.go
index a437e88..f4e9cb0 100644
--- a/archive.go
+++ b/archive.go
@@ -31,6 +31,7 @@ func handleGetArchive(w http.ResponseWriter, r *gitRequest) {
format = "tar.bz2"
default:
fail500(w, "handleGetArchive", errors.New("invalid archive format"))
+ return
}
archiveFilename := path.Base(r.ArchivePath)
@@ -53,6 +54,7 @@ func handleGetArchive(w http.ResponseWriter, r *gitRequest) {
tempFile, err := prepareArchiveTempfile(path.Dir(r.ArchivePath), archiveFilename)
if err != nil {
fail500(w, "handleGetArchive create tempfile for archive", err)
+ return
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
@@ -91,7 +93,7 @@ func handleGetArchive(w http.ResponseWriter, r *gitRequest) {
}
defer compressCmd.Wait()
- archiveStdout.Close()
+ //archiveStdout.Close()
}
// Every Read() from stdout will be synchronously written to tempFile
// before it comes out the TeeReader.
@@ -102,6 +104,9 @@ func handleGetArchive(w http.ResponseWriter, r *gitRequest) {
w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
if _, err := io.Copy(w, archiveReader); err != nil {
logContext("handleGetArchive read from subprocess", err)
+ if compressCmd != nil {
+ cleanUpProcessGroup(compressCmd)
+ }
return
}
if err := archiveCmd.Wait(); err != nil {
diff --git a/helpers.go b/helpers.go
index 467c447..6fe6f95 100644
--- a/helpers.go
+++ b/helpers.go
@@ -54,8 +54,9 @@ func cleanUpProcessGroup(cmd *exec.Cmd) {
process := cmd.Process
if process != nil && process.Pid > 0 {
+ log.Printf("killing %s: %d", cmd.Path, process.Pid)
// Send SIGTERM to the process group of cmd
- syscall.Kill(-process.Pid, syscall.SIGTERM)
+ syscall.Kill(process.Pid, syscall.SIGTERM)
}
// reap our child process