Commit 8ef73396 authored by Steve Azzopardi's avatar Steve Azzopardi

Merge branch 'extract-docker-volume-operations-from-executor' into 'master'

Extract volumes configuration to a separate struct

See merge request !1261
parents 90331ed9 4eec4915
......@@ -159,6 +159,7 @@ mocks: $(MOCKERY)
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./vendor/github.com/ayufan/golang-kardianos-service -output=./helpers/service/mocks -name='(Interface)'
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./helpers/docker -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./helpers/certificate -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./executors/docker -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./cache -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./common -all -inpkg
GOPATH=$(ORIGINAL_GOPATH) mockery $(MOCKERY_FLAGS) -dir=./log -all -inpkg
......
This diff is collapsed.
This diff is collapsed.
package volumes
import (
"context"
"fmt"
"sync"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
)
type containerClient interface {
docker_helpers.Client
LabelContainer(container *container.Config, containerType string, otherLabels ...string)
WaitForContainer(id string) error
RemoveContainer(ctx context.Context, id string) error
}
type CacheContainersManager interface {
FindOrCleanExisting(containerName string, containerPath string) string
Create(containerName string, containerPath string) (string, error)
Cleanup(ctx context.Context, ids []string) chan bool
}
type cacheContainerManager struct {
ctx context.Context
logger debugLogger
containerClient containerClient
helperImage *types.ImageInspect
outdatedHelperImage bool
failedContainerIDs []string
}
func NewCacheContainerManager(ctx context.Context, logger debugLogger, cClient containerClient, helperImage *types.ImageInspect, outdatedHelperImage bool) CacheContainersManager {
return &cacheContainerManager{
ctx: ctx,
logger: logger,
containerClient: cClient,
helperImage: helperImage,
outdatedHelperImage: outdatedHelperImage,
}
}
func (m *cacheContainerManager) FindOrCleanExisting(containerName string, containerPath string) string {
inspected, err := m.containerClient.ContainerInspect(m.ctx, containerName)
if err != nil {
m.logger.Debugln(fmt.Sprintf("Error while inspecting %q container: %v", containerName, err))
return ""
}
// check if we have valid cache, if not remove the broken container
_, ok := inspected.Config.Volumes[containerPath]
if !ok {
m.logger.Debugln(fmt.Sprintf("Removing broken cache container for %q path", containerPath))
err = m.containerClient.RemoveContainer(m.ctx, inspected.ID)
m.logger.Debugln(fmt.Sprintf("Cache container for %q path removed with %v", containerPath, err))
return ""
}
return inspected.ID
}
func (m *cacheContainerManager) Create(containerName string, containerPath string) (string, error) {
containerID, err := m.createCacheContainer(containerName, containerPath)
if err != nil {
return "", err
}
err = m.startCacheContainer(containerID)
if err != nil {
return "", err
}
return containerID, nil
}
func (m *cacheContainerManager) createCacheContainer(containerName string, containerPath string) (string, error) {
config := &container.Config{
Image: m.helperImage.ID,
Cmd: m.getCacheCommand(containerPath),
Volumes: map[string]struct{}{
containerPath: {},
},
}
m.containerClient.LabelContainer(config, "cache", "cache.dir="+containerPath)
hostConfig := &container.HostConfig{
LogConfig: container.LogConfig{
Type: "json-file",
},
}
resp, err := m.containerClient.ContainerCreate(m.ctx, config, hostConfig, nil, containerName)
if err != nil {
if resp.ID != "" {
m.failedContainerIDs = append(m.failedContainerIDs, resp.ID)
}
return "", err
}
return resp.ID, nil
}
func (m *cacheContainerManager) getCacheCommand(containerPath string) []string {
// TODO: Remove in 12.0 to start using the command from `gitlab-runner-helper`
if m.outdatedHelperImage {
m.logger.Debugln("Falling back to old gitlab-runner-cache command")
return []string{"gitlab-runner-cache", containerPath}
}
return []string{"gitlab-runner-helper", "cache-init", containerPath}
}
func (m *cacheContainerManager) startCacheContainer(containerID string) error {
m.logger.Debugln(fmt.Sprintf("Starting cache container %q...", containerID))
err := m.containerClient.ContainerStart(m.ctx, containerID, types.ContainerStartOptions{})
if err != nil {
m.failedContainerIDs = append(m.failedContainerIDs, containerID)
return err
}
m.logger.Debugln(fmt.Sprintf("Waiting for cache container %q...", containerID))
err = m.containerClient.WaitForContainer(containerID)
if err != nil {
m.failedContainerIDs = append(m.failedContainerIDs, containerID)
return err
}
return nil
}
func (m *cacheContainerManager) Cleanup(ctx context.Context, ids []string) chan bool {
done := make(chan bool, 1)
ids = append(m.failedContainerIDs, ids...)
go func() {
wg := new(sync.WaitGroup)
wg.Add(len(ids))
for _, id := range ids {
m.remove(ctx, wg, id)
}
wg.Wait()
done <- true
}()
return done
}
func (m *cacheContainerManager) remove(ctx context.Context, wg *sync.WaitGroup, id string) {
go func() {
err := m.containerClient.RemoveContainer(ctx, id)
if err != nil {
m.logger.Debugln(fmt.Sprintf("Error while removing the container: %v", err))
}
wg.Done()
}()
}
package volumes
import (
"context"
"errors"
"fmt"
"testing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestNewCacheContainerManager(t *testing.T) {
logger := newDebugLoggerMock()
m := NewCacheContainerManager(context.Background(), logger, nil, nil, true)
assert.IsType(t, &cacheContainerManager{}, m)
}
func getCacheContainerManager() (*cacheContainerManager, *mockContainerClient) {
cClient := new(mockContainerClient)
m := &cacheContainerManager{
logger: newDebugLoggerMock(),
containerClient: cClient,
failedContainerIDs: make([]string, 0),
helperImage: &types.ImageInspect{ID: "helper-image"},
outdatedHelperImage: false,
}
return m, cClient
}
func TestCacheContainerManager_FindExistingCacheContainer(t *testing.T) {
containerName := "container-name"
containerPath := "container-path"
testCases := map[string]struct {
inspectResult types.ContainerJSON
inspectError error
expectedContainerID string
expectedRemoveID string
}{
"error on container inspection": {
inspectError: errors.New("test error"),
expectedContainerID: "",
},
"container with valid cache exists": {
inspectResult: types.ContainerJSON{
ContainerJSONBase: &types.ContainerJSONBase{
ID: "existingWithValidCacheID",
},
Config: &container.Config{
Volumes: map[string]struct{}{
containerPath: {},
},
},
},
inspectError: nil,
expectedContainerID: "existingWithValidCacheID",
},
"container without valid cache exists": {
inspectResult: types.ContainerJSON{
ContainerJSONBase: &types.ContainerJSONBase{
ID: "existingWithInvalidCacheID",
},
Config: &container.Config{
Volumes: map[string]struct{}{
"different-path": {},
},
},
},
inspectError: nil,
expectedContainerID: "",
expectedRemoveID: "existingWithInvalidCacheID",
},
}
for testName, testCase := range testCases {
t.Run(testName, func(t *testing.T) {
m, cClient := getCacheContainerManager()
defer cClient.AssertExpectations(t)
cClient.On("ContainerInspect", mock.Anything, containerName).
Return(testCase.inspectResult, testCase.inspectError).
Once()
if testCase.expectedRemoveID != "" {
cClient.On("RemoveContainer", mock.Anything, testCase.expectedRemoveID).
Return(nil).
Once()
}
containerID := m.FindOrCleanExisting(containerName, containerPath)
assert.Equal(t, testCase.expectedContainerID, containerID)
})
}
}
func TestCacheContainerManager_CreateCacheContainer(t *testing.T) {
containerName := "container-name"
containerPath := "container-path"
testCases := map[string]struct {
expectedContainerID string
createResult container.ContainerCreateCreatedBody
createError error
containerID string
startError error
waitForContainerError error
expectedFailedContainerID string
expectedError error
}{
"error on container create": {
createError: errors.New("test error"),
expectedError: errors.New("test error"),
},
"error on container create with returnedID": {
createResult: container.ContainerCreateCreatedBody{
ID: "containerID",
},
createError: errors.New("test error"),
expectedFailedContainerID: "containerID",
expectedError: errors.New("test error"),
},
"error on container start": {
createResult: container.ContainerCreateCreatedBody{
ID: "containerID",
},
containerID: "containerID",
startError: errors.New("test error"),
expectedFailedContainerID: "containerID",
expectedError: errors.New("test error"),
},
"error on wait for container": {
createResult: container.ContainerCreateCreatedBody{
ID: "containerID",
},
containerID: "containerID",
waitForContainerError: errors.New("test error"),
expectedFailedContainerID: "containerID",
expectedError: errors.New("test error"),
},
"success": {
createResult: container.ContainerCreateCreatedBody{
ID: "containerID",
},
containerID: "containerID",
expectedContainerID: "containerID",
expectedError: nil,
},
}
// TODO: Remove in 12.0
outdatedHelperImageValues := map[bool][]string{
true: {"gitlab-runner-cache", "container-path"},
false: {"gitlab-runner-helper", "cache-init", "container-path"},
}
for testName, testCase := range testCases {
for outdatedHelperImage, expectedCommand := range outdatedHelperImageValues {
t.Run(fmt.Sprintf("%s-outdated-helper-image-is-%v", testName, outdatedHelperImage), func(t *testing.T) {
m, cClient := getCacheContainerManager()
m.outdatedHelperImage = outdatedHelperImage
defer cClient.AssertExpectations(t)
configMatcher := mock.MatchedBy(func(config *container.Config) bool {
if config.Image != "helper-image" {
return false
}
if len(config.Cmd) != len(expectedCommand) {
return false
}
return config.Cmd[0] == expectedCommand[0]
})
cClient.On("LabelContainer", configMatcher, "cache", fmt.Sprintf("cache.dir=%s", containerPath)).
Once()
cClient.On("ContainerCreate", mock.Anything, configMatcher, mock.Anything, mock.Anything, containerName).
Return(testCase.createResult, testCase.createError).
Once()
if testCase.createError == nil {
cClient.On("ContainerStart", mock.Anything, testCase.containerID, mock.Anything).
Return(testCase.startError).
Once()
if testCase.startError == nil {
cClient.On("WaitForContainer", testCase.containerID).
Return(testCase.waitForContainerError).
Once()
}
}
require.Empty(t, m.failedContainerIDs, "Initial list of failed containers should be empty")
containerID, err := m.Create(containerName, containerPath)
assert.Equal(t, err, testCase.expectedError)
assert.Equal(t, testCase.expectedContainerID, containerID)
if testCase.expectedFailedContainerID != "" {
assert.Len(t, m.failedContainerIDs, 1)
assert.Contains(
t, m.failedContainerIDs, testCase.expectedFailedContainerID,
"List of failed container should be updated with %s", testCase.expectedContainerID,
)
} else {
assert.Empty(t, m.failedContainerIDs, "List of failed containers should not be updated")
}
})
}
}
}
func TestCacheContainerManager_Cleanup(t *testing.T) {
ctx := context.Background()
containerClientMock := new(mockContainerClient)
defer containerClientMock.AssertExpectations(t)
loggerMock := new(mockDebugLogger)
defer loggerMock.AssertExpectations(t)
containerClientMock.On("RemoveContainer", ctx, "failed-container-1").
Return(nil).
Once()
containerClientMock.On("RemoveContainer", ctx, "container-1-with-remove-error").
Return(errors.New("test-error")).
Once()
containerClientMock.On("RemoveContainer", ctx, "container-1").
Return(nil).
Once()
loggerMock.On("Debugln", "Error while removing the container: test-error").
Once()
m := &cacheContainerManager{
containerClient: containerClientMock,
logger: loggerMock,
failedContainerIDs: []string{"failed-container-1", "container-1-with-remove-error"},
}
done := m.Cleanup(ctx, []string{"container-1"})
<-done
}
package volumes
import (
"context"
"errors"
"fmt"
"path"
"path/filepath"
"strings"
)
var ErrCacheVolumesDisabled = errors.New("cache volumes feature disabled")
type Manager interface {
Create(volume string) error
CreateTemporary(containerPath string) error
Binds() []string
ContainerIDs() []string
Cleanup(ctx context.Context) chan bool
}
type ManagerConfig struct {
CacheDir string
BaseContainerPath string
UniqueName string
DisableCache bool
}
type manager struct {
config ManagerConfig
logger debugLogger
cacheContainersManager CacheContainersManager
volumeBindings []string
cacheContainerIDs []string
tmpContainerIDs []string
managedVolumes pathList
}
func NewManager(logger debugLogger, ccManager CacheContainersManager, config ManagerConfig) Manager {
return &manager{
config: config,
logger: logger,
cacheContainersManager: ccManager,
volumeBindings: make([]string, 0),
cacheContainerIDs: make([]string, 0),
tmpContainerIDs: make([]string, 0),
managedVolumes: pathList{},
}
}
func (m *manager) Create(volume string) error {
if len(volume) < 1 {
return nil
}
hostVolume := strings.SplitN(volume, ":", 2)
var err error
switch len(hostVolume) {
case 2:
err = m.addHostVolume(hostVolume[0], hostVolume[1])
case 1:
err = m.addCacheVolume(hostVolume[0])
}
return err
}
func (m *manager) addHostVolume(hostPath string, containerPath string) error {
containerPath = m.getAbsoluteContainerPath(containerPath)
err := m.managedVolumes.Add(containerPath)
if err != nil {
return err
}
m.appendVolumeBind(hostPath, containerPath)
return nil
}
func (m *manager) getAbsoluteContainerPath(dir string) string {
if path.IsAbs(dir) {
return dir
}
return path.Join(m.config.BaseContainerPath, dir)
}
func (m *manager) appendVolumeBind(hostPath string, containerPath string) {
m.logger.Debugln(fmt.Sprintf("Using host-based %q for %q...", hostPath, containerPath))
bindDefinition := fmt.Sprintf("%v:%v", filepath.ToSlash(hostPath), containerPath)
m.volumeBindings = append(m.volumeBindings, bindDefinition)
}
func (m *manager) addCacheVolume(containerPath string) error {
// disable cache for automatic container cache,
// but leave it for host volumes (they are shared on purpose)
if m.config.DisableCache {
m.logger.Debugln("Cache containers feature is disabled")
return ErrCacheVolumesDisabled
}
if m.config.CacheDir != "" {
return m.createHostBasedCacheVolume(containerPath)
}
_, err := m.createContainerBasedCacheVolume(containerPath)
return err
}
func (m *manager) createHostBasedCacheVolume(containerPath string) error {
containerPath = m.getAbsoluteContainerPath(containerPath)
err := m.managedVolumes.Add(containerPath)
if err != nil {
return err
}
hostPath := fmt.Sprintf("%s/%s/%s", m.config.CacheDir, m.config.UniqueName, hashContainerPath(containerPath))
hostPath, err = filepath.Abs(hostPath)
if err != nil {
return err
}
m.appendVolumeBind(hostPath, containerPath)
return nil
}
func (m *manager) createContainerBasedCacheVolume(containerPath string) (string, error) {
containerPath = m.getAbsoluteContainerPath(containerPath)
err := m.managedVolumes.Add(containerPath)
if err != nil {
return "", err
}
containerName := fmt.Sprintf("%s-cache-%s", m.config.UniqueName, hashContainerPath(containerPath))
containerID := m.cacheContainersManager.FindOrCleanExisting(containerName, containerPath)
// create new cache container for that project
if containerID == "" {
var err error
containerID, err = m.cacheContainersManager.Create(containerName, containerPath)
if err != nil {
return "", err
}
}
m.logger.Debugln(fmt.Sprintf("Using container %q as cache %q...", containerID, containerPath))
m.cacheContainerIDs = append(m.cacheContainerIDs, containerID)
return containerID, nil
}
func (m *manager) CreateTemporary(containerPath string) error {
id, err := m.createContainerBasedCacheVolume(containerPath)
if err != nil {
return err
}
m.tmpContainerIDs = append(m.tmpContainerIDs, id)
return nil
}
func (m *manager) Binds() []string {
return m.volumeBindings
}
func (m *manager) ContainerIDs() []string {
return m.cacheContainerIDs
}
func (m *manager) Cleanup(ctx context.Context) chan bool {
return m.cacheContainersManager.Cleanup(ctx, m.tmpContainerIDs)
}
This diff is collapsed.
// Code generated by mockery v1.0.0. DO NOT EDIT.
package volumes
import context "context"
import mock "github.com/stretchr/testify/mock"
// MockCacheContainersManager is an autogenerated mock type for the CacheContainersManager type
type MockCacheContainersManager struct {
mock.Mock
}
// Cleanup provides a mock function with given fields: ctx, ids
func (_m *MockCacheContainersManager) Cleanup(ctx context.Context, ids []string) chan bool {
ret := _m.Called(ctx, ids)
var r0 chan bool
if rf, ok := ret.Get(0).(func(context.Context, []string) chan bool); ok {
r0 = rf(ctx, ids)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(chan bool)
}
}
return r0
}
// Create provides a mock function with given fields: containerName, containerPath
func (_m *MockCacheContainersManager) Create(containerName string, containerPath string) (string, error) {
ret := _m.Called(containerName, containerPath)
var r0 string
if rf, ok := ret.Get(0).(func(string, string) string); ok {
r0 = rf(containerName, containerPath)
} else {
r0 = ret.Get(0).(string)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string) error); ok {
r1 = rf(containerName, containerPath)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FindOrCleanExisting provides a mock function with given fields: containerName, containerPath
func (_m *MockCacheContainersManager) FindOrCleanExisting(containerName string, containerPath string) string {
ret := _m.Called(containerName, containerPath)
var r0 string
if rf, ok := ret.Get(0).(func(string, string) string); ok {
r0 = rf(containerName, containerPath)
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Code generated by mockery v1.0.0. DO NOT EDIT.
package volumes
import context "context"
import mock "github.com/stretchr/testify/mock"