Commit 8f1722e1 authored by Steve Azzopardi's avatar Steve Azzopardi Committed by Steve Azzopardi

Merge branch 'support-windows-docker-volumes-configuration' into 'master'

Support windows docker volumes configuration

Closes #3909

See merge request gitlab-org/gitlab-runner!1269
parent b3b26bcc
......@@ -26,6 +26,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
docker_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/helperimage"
......@@ -1073,6 +1074,7 @@ func (e *executor) createBuildVolume() error {
return nil
}
func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
e.SetCurrentStage(DockerExecutorStagePrepare)
......@@ -1080,12 +1082,19 @@ func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
return errors.New("missing docker configuration")
}
err := e.prepareBuildsDir(options)
e.AbstractExecutor.PrepareConfiguration(options)
err := e.connectDocker()
if err != nil {
return err
}
err = e.AbstractExecutor.Prepare(options)
err = e.prepareBuildsDir(options)
if err != nil {
return err
}
err = e.AbstractExecutor.PrepareBuildAndShell()
if err != nil {
return err
}
......@@ -1101,11 +1110,6 @@ func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
e.Println("Using Docker executor with image", imageName, "...")
err = e.connectDocker()
if err != nil {
return err
}
err = e.createDependencies()
if err != nil {
return err
......@@ -1120,12 +1124,22 @@ var (
)
func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error {
volumeParser, err := parser.New(e.info)
if err != nil {
return fmt.Errorf("couldn't create volumes parser: %v", err)
}
isHostMounted, err := volumes.IsHostMountedVolume(volumeParser, e.RootDir(), options.Config.Docker.Volumes...)
if err != nil {
return err
}
// We need to set proper value for e.SharedBuildsDir because
// it's required to properly start the job, what is done inside of
// e.AbstractExecutor.Prepare()
// And a started job is required for Volumes Manager to work, so it's
// done before the manager is even created.
if volumes.IsHostMountedVolume(e.RootDir(), options.Config.Docker.Volumes...) {
if isHostMounted {
e.SharedBuildsDir = true
}
......
......@@ -24,6 +24,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
docker_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/helperimage"
......@@ -566,39 +567,55 @@ func TestDockerGetExistingDockerImageIfPullFails(t *testing.T) {
func TestPrepareBuildsDir(t *testing.T) {
tests := map[string]struct {
osType string
rootDir string
volumes []string
expectedSharedBuildsDir bool
expectedError error
}{
"rootDir mounted as host based volume": {
osType: parser.OSTypeLinux,
rootDir: "/build",
volumes: []string{"/build:/build"},
expectedSharedBuildsDir: true,
},
"rootDir mounted as container based volume": {
osType: parser.OSTypeLinux,
rootDir: "/build",
volumes: []string{"/build"},
expectedSharedBuildsDir: false,
},
"rootDir not mounted as volume": {
osType: parser.OSTypeLinux,
rootDir: "/build",
volumes: []string{"/folder:/folder"},
expectedSharedBuildsDir: false,
},
"rootDir's parent mounted as volume": {
osType: parser.OSTypeLinux,
rootDir: "/build/other/directory",
volumes: []string{"/build/:/build"},
expectedSharedBuildsDir: true,
},
"rootDir is not an absolute path": {
osType: parser.OSTypeLinux,
rootDir: "builds",
expectedError: buildDirectoryNotAbsoluteErr,
},
"rootDir is /": {
osType: parser.OSTypeLinux,
rootDir: "/",
expectedError: buildDirectoryIsRootPathErr,
},
"error on volume parsing": {
osType: parser.OSTypeLinux,
rootDir: "/build",
volumes: []string{""},
expectedError: parser.NewInvalidVolumeSpecErr(""),
},
"error on volume parser creation": {
expectedError: errors.New(`couldn't create volumes parser: unsupported OSType ""`),
},
}
for testName, test := range tests {
......@@ -620,7 +637,11 @@ func TestPrepareBuildsDir(t *testing.T) {
AbstractExecutor: executors.AbstractExecutor{
Config: c,
},
info: types.Info{
OSType: test.osType,
},
}
err := e.prepareBuildsDir(options)
assert.Equal(t, test.expectedError, err)
assert.Equal(t, test.expectedSharedBuildsDir, e.SharedBuildsDir)
......
......@@ -4,9 +4,9 @@ import (
"context"
"errors"
"fmt"
"path"
"path/filepath"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
)
var ErrCacheVolumesDisabled = errors.New("cache volumes feature disabled")
......@@ -29,6 +29,7 @@ type ManagerConfig struct {
type manager struct {
config ManagerConfig
logger debugLogger
parser parser.Parser
cacheContainersManager CacheContainersManager
......@@ -39,10 +40,11 @@ type manager struct {
managedVolumes pathList
}
func NewManager(logger debugLogger, ccManager CacheContainersManager, config ManagerConfig) Manager {
func NewManager(logger debugLogger, volumeParser parser.Parser, ccManager CacheContainersManager, config ManagerConfig) Manager {
return &manager{
config: config,
logger: logger,
parser: volumeParser,
cacheContainersManager: ccManager,
volumeBindings: make([]string, 0),
cacheContainerIDs: make([]string, 0),
......@@ -56,48 +58,48 @@ func (m *manager) Create(volume string) error {
return nil
}
hostVolume := strings.SplitN(volume, ":", 2)
parsedVolume, err := m.parser.ParseVolume(volume)
if err != nil {
return err
}
var err error
switch len(hostVolume) {
switch parsedVolume.Len() {
case 2:
err = m.addHostVolume(hostVolume[0], hostVolume[1])
err = m.addHostVolume(parsedVolume)
case 1:
err = m.addCacheVolume(hostVolume[0])
err = m.addCacheVolume(parsedVolume)
}
return err
}
func (m *manager) addHostVolume(hostPath string, containerPath string) error {
containerPath = m.getAbsoluteContainerPath(containerPath)
func (m *manager) addHostVolume(volume *parser.Volume) error {
volume.Destination = m.getAbsoluteContainerPath(volume.Destination)
err := m.managedVolumes.Add(containerPath)
err := m.managedVolumes.Add(volume.Destination)
if err != nil {
return err
}
m.appendVolumeBind(hostPath, containerPath)
m.appendVolumeBind(volume)
return nil
}
func (m *manager) getAbsoluteContainerPath(dir string) string {
if path.IsAbs(dir) {
if filepath.IsAbs(dir) {
return dir
}
return path.Join(m.config.BaseContainerPath, dir)
return filepath.Join(m.config.BaseContainerPath, dir)
}
func (m *manager) appendVolumeBind(hostPath string, containerPath string) {
m.logger.Debugln(fmt.Sprintf("Using host-based %q for %q...", hostPath, containerPath))
bindDefinition := fmt.Sprintf("%v:%v", filepath.ToSlash(hostPath), containerPath)
m.volumeBindings = append(m.volumeBindings, bindDefinition)
func (m *manager) appendVolumeBind(volume *parser.Volume) {
m.logger.Debugln(fmt.Sprintf("Using host-based %q for %q...", volume.Source, volume.Destination))
m.volumeBindings = append(m.volumeBindings, volume.Definition())
}
func (m *manager) addCacheVolume(containerPath string) error {
func (m *manager) addCacheVolume(volume *parser.Volume) error {
// disable cache for automatic container cache,
// but leave it for host volumes (they are shared on purpose)
if m.config.DisableCache {
......@@ -107,10 +109,10 @@ func (m *manager) addCacheVolume(containerPath string) error {
}
if m.config.CacheDir != "" {
return m.createHostBasedCacheVolume(containerPath)
return m.createHostBasedCacheVolume(volume.Destination)
}
_, err := m.createContainerBasedCacheVolume(containerPath)
_, err := m.createContainerBasedCacheVolume(volume.Destination)
return err
}
......@@ -123,13 +125,16 @@ func (m *manager) createHostBasedCacheVolume(containerPath string) error {
return err
}
hostPath := fmt.Sprintf("%s/%s/%s", m.config.CacheDir, m.config.UniqueName, hashContainerPath(containerPath))
hostPath := filepath.Join(m.config.CacheDir, m.config.UniqueName, hashContainerPath(containerPath))
hostPath, err = filepath.Abs(hostPath)
if err != nil {
return err
}
m.appendVolumeBind(hostPath, containerPath)
m.appendVolumeBind(&parser.Volume{
Source: hostPath,
Destination: containerPath,
})
return nil
}
......
......@@ -8,6 +8,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
)
func newDebugLoggerMock() *mockDebugLogger {
......@@ -25,7 +27,7 @@ func TestErrVolumeAlreadyDefined(t *testing.T) {
func TestNewDefaultManager(t *testing.T) {
logger := newDebugLoggerMock()
m := NewManager(logger, nil, ManagerConfig{})
m := NewManager(logger, nil, nil, ManagerConfig{})
assert.IsType(t, &manager{}, m)
}
......@@ -47,9 +49,17 @@ func addCacheContainerManager(manager *manager) *MockCacheContainersManager {
return containerManager
}
func addParser(manager *manager) *parser.MockParser {
parserMock := new(parser.MockParser)
manager.parser = parserMock
return parserMock
}
func TestDefaultManager_CreateUserVolumes_HostVolume(t *testing.T) {
testCases := map[string]struct {
volume string
parsedVolume *parser.Volume
baseContainerPath string
expectedBinding []string
expectedError error
......@@ -60,24 +70,29 @@ func TestDefaultManager_CreateUserVolumes_HostVolume(t *testing.T) {
},
"volume with absolute path": {
volume: "/host:/volume",
parsedVolume: &parser.Volume{Source: "/host", Destination: "/volume"},
expectedBinding: []string{"/host:/duplicated", "/host:/volume"},
},
"volume with absolute path and with baseContainerPath specified": {
volume: "/host:/volume",
parsedVolume: &parser.Volume{Source: "/host", Destination: "/volume"},
baseContainerPath: "/builds",
expectedBinding: []string{"/host:/duplicated", "/host:/volume"},
},
"volume without absolute path and without baseContainerPath specified": {
volume: "/host:volume",
parsedVolume: &parser.Volume{Source: "/host", Destination: "volume"},
expectedBinding: []string{"/host:/duplicated", "/host:volume"},
},
"volume without absolute path and with baseContainerPath specified": {
volume: "/host:volume",
parsedVolume: &parser.Volume{Source: "/host", Destination: "volume"},
baseContainerPath: "/builds/project",
expectedBinding: []string{"/host:/duplicated", "/host:/builds/project/volume"},
},
"duplicated volume specification": {
volume: "/host/new:/duplicated",
parsedVolume: &parser.Volume{Source: "/host/new", Destination: "/duplicated"},
expectedBinding: []string{"/host:/duplicated"},
expectedError: NewErrVolumeAlreadyDefined("/duplicated"),
},
......@@ -91,9 +106,22 @@ func TestDefaultManager_CreateUserVolumes_HostVolume(t *testing.T) {
m := newDefaultManager(config)
volumeParser := addParser(m)
defer volumeParser.AssertExpectations(t)
volumeParser.On("ParseVolume", "/host:/duplicated").
Return(&parser.Volume{Source: "/host", Destination: "/duplicated"}, nil).
Once()
err := m.Create("/host:/duplicated")
require.NoError(t, err)
if len(testCase.volume) > 0 {
volumeParser.On("ParseVolume", testCase.volume).
Return(testCase.parsedVolume, nil).
Once()
}
err = m.Create(testCase.volume)
assert.Equal(t, testCase.expectedError, err)
assert.Equal(t, testCase.expectedBinding, m.volumeBindings)
......@@ -106,6 +134,7 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_Disabled(t *testing.T) {
testCases := map[string]struct {
volume string
parsedVolume *parser.Volume
baseContainerPath string
expectedCacheContainerIDs []string
......@@ -117,25 +146,30 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_Disabled(t *testing.T) {
},
"volume with absolute path, without baseContainerPath and with disableCache": {
volume: "/volume",
parsedVolume: &parser.Volume{Destination: "/volume"},
baseContainerPath: "",
expectedError: ErrCacheVolumesDisabled,
},
"volume with absolute path, with baseContainerPath and with disableCache": {
volume: "/volume",
parsedVolume: &parser.Volume{Destination: "/volume"},
baseContainerPath: "/builds/project",
expectedError: ErrCacheVolumesDisabled,
},
"volume without absolute path, without baseContainerPath and with disableCache": {
volume: "volume",
parsedVolume: &parser.Volume{Destination: "volume"},
expectedError: ErrCacheVolumesDisabled,
},
"volume without absolute path, with baseContainerPath and with disableCache": {
volume: "volume",
parsedVolume: &parser.Volume{Destination: "volume"},
baseContainerPath: "/builds/project",
expectedError: ErrCacheVolumesDisabled,
},
"duplicated volume definition": {
volume: "/duplicated",
parsedVolume: &parser.Volume{Destination: "/duplicated"},
baseContainerPath: "",
expectedError: ErrCacheVolumesDisabled,
},
......@@ -150,9 +184,22 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_Disabled(t *testing.T) {
m := newDefaultManager(config)
volumeParser := addParser(m)
defer volumeParser.AssertExpectations(t)
volumeParser.On("ParseVolume", "/host:/duplicated").
Return(&parser.Volume{Source: "/host", Destination: "/duplicated"}, nil).
Once()
err := m.Create("/host:/duplicated")
require.NoError(t, err)
if len(testCase.volume) > 0 {
volumeParser.On("ParseVolume", testCase.volume).
Return(testCase.parsedVolume, nil).
Once()
}
err = m.Create(testCase.volume)
assert.Equal(t, testCase.expectedError, err)
assert.Equal(t, expectedBinding, m.volumeBindings)
......@@ -218,9 +265,20 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_HostBased(t *testing.T) {
m := newDefaultManager(config)
volumeParser := addParser(m)
defer volumeParser.AssertExpectations(t)
volumeParser.On("ParseVolume", "/host:/duplicated").
Return(&parser.Volume{Source: "/host", Destination: "/duplicated"}, nil).
Once()
err := m.Create("/host:/duplicated")
require.NoError(t, err)
volumeParser.On("ParseVolume", testCase.volume).
Return(&parser.Volume{Destination: testCase.volume}, nil).
Once()
err = m.Create(testCase.volume)
assert.Equal(t, testCase.expectedError, err)
assert.Equal(t, testCase.expectedBinding, m.volumeBindings)
......@@ -314,8 +372,16 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_ContainerBased(t *testing.
m := newDefaultManager(config)
containerManager := addCacheContainerManager(m)
volumeParser := addParser(m)
defer containerManager.AssertExpectations(t)
defer func() {
containerManager.AssertExpectations(t)
volumeParser.AssertExpectations(t)
}()
volumeParser.On("ParseVolume", "/host:/duplicated").
Return(&parser.Volume{Source: "/host", Destination: "/duplicated"}, nil).
Once()
err := m.Create("/host:/duplicated")
require.NoError(t, err)
......@@ -332,6 +398,10 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_ContainerBased(t *testing.
}
}
volumeParser.On("ParseVolume", testCase.volume).
Return(&parser.Volume{Destination: testCase.volume}, nil).
Once()
err = m.Create(testCase.volume)
assert.Equal(t, testCase.expectedError, err)
......@@ -350,8 +420,12 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_ContainerBased_WithError(t
m := newDefaultManager(config)
containerManager := addCacheContainerManager(m)
volumeParser := addParser(m)
defer containerManager.AssertExpectations(t)
defer func() {
containerManager.AssertExpectations(t)
volumeParser.AssertExpectations(t)
}()
containerManager.On("FindOrCleanExisting", "unique-cache-f69aef9fb01e88e6213362a04877452d", "/builds/project/volume").
Return("").
......@@ -361,6 +435,24 @@ func TestDefaultManager_CreateUserVolumes_CacheVolume_ContainerBased_WithError(t
Return("", errors.New("test error")).
Once()
volumeParser.On("ParseVolume", "volume").
Return(&parser.Volume{Destination: "volume"}, nil).
Once()
err := m.Create("volume")
assert.Error(t, err)
}
func TestDefaultManager_CreateUserVolumes_ParserError(t *testing.T) {
m := newDefaultManager(ManagerConfig{})
volumeParser := addParser(m)
defer volumeParser.AssertExpectations(t)
volumeParser.On("ParseVolume", "volume").
Return(nil, errors.New("parser-test-error")).
Once()
err := m.Create("volume")
assert.Error(t, err)
}
......@@ -369,6 +461,7 @@ func TestDefaultManager_CreateTemporary(t *testing.T) {
testCases := map[string]struct {
volume string
newContainerID string
returnedParsedVolume *parser.Volume
containerCreateError error
expectedContainerName string
expectedContainerPath string
......@@ -378,6 +471,7 @@ func TestDefaultManager_CreateTemporary(t *testing.T) {
}{
"volume created": {
volume: "volume",
returnedParsedVolume: &parser.Volume{Destination: "volume"},
newContainerID: "newContainerID",
expectedContainerName: "uniq-cache-f69aef9fb01e88e6213362a04877452d",
expectedContainerPath: "/builds/project/volume",
......@@ -386,6 +480,7 @@ func TestDefaultManager_CreateTemporary(t *testing.T) {
},
"cache container creation error": {
volume: "volume",
returnedParsedVolume: &parser.Volume{Destination: "volume"},
newContainerID: "",
containerCreateError: errors.New("test-error"),
expectedError: errors.New("test-error"),
......@@ -405,8 +500,16 @@ func TestDefaultManager_CreateTemporary(t *testing.T) {
m := newDefaultManager(config)
containerManager := addCacheContainerManager(m)
volumeParser := addParser(m)
defer func() {
containerManager.AssertExpectations(t)
volumeParser.AssertExpectations(t)
}()
defer containerManager.AssertExpectations(t)
volumeParser.On("ParseVolume", "/host:/duplicated").
Return(&parser.Volume{Source: "/host", Destination: "/duplicated"}, nil).
Once()
err := m.Create("/host:/duplicated")
require.NoError(t, err)
......
package parser
import (
"regexp"
"strings"
)
type baseParser struct{}
// The way how matchesToVolumeSpecParts parses the volume mount specification and assigns
// parts was inspired by how Docker Engine's `windowsParser` is created. The original sources
// can be found at:
//
// https://github.com/docker/engine/blob/a79fabbfe84117696a19671f4aa88b82d0f64fc1/volume/mounts/windows_parser.go
//
// The original source is licensed under Apache License 2.0 and the copyright for it
// goes to Docker, Inc.
func (p *baseParser) matchesToVolumeSpecParts(spec string, specExp *regexp.Regexp) (map[string]string, error) {
match := specExp.FindStringSubmatch(strings.ToLower(spec))
if len(match) == 0 {
return nil, NewInvalidVolumeSpecErr(spec)
}
matchgroups := make(map[string]string)
for i, name := range specExp.SubexpNames() {
matchgroups[name] = strings.ToLower(match[i])
}
parts := map[string]string{
"source": "",
"destination": "",
"mode": "",
}
for group := range parts {
content, ok := matchgroups[group]
if ok {
parts[group] = content
}
}
return parts, nil
}
package parser
import (
"fmt"
)
type InvalidVolumeSpecError struct {
spec string
}
func (e *InvalidVolumeSpecError) Error() string {
return fmt.Sprintf("invalid volume specification: %q", e.spec)
}
func NewInvalidVolumeSpecErr(spec string) error {
return &InvalidVolumeSpecError{
spec: spec,
}
}
package parser
import (
"regexp"
)
const (
linuxDir = `/(?:[^\\/:*?"<>|\r\n ]+/?)*`
linuxVolumeName = `[^\\/:*?"<>|\r\n]+`
linuxSource = `((?P<source>((` + linuxDir + `)|(` + linuxVolumeName + `))):)?`
linuxDestination = `(?P<destination>(?:` + linuxDir + `))`
linuxMode = `(:(?P<mode>(?i)ro|rw))?`
)
type linuxParser struct {
baseParser
}
func newLinuxParser() Parser {
return new(linuxParser)
}
func (p *linuxParser) ParseVolume(spec string) (*Volume, error) {
specExp := regexp.MustCompile(`^` + linuxSource + linuxDestination + linuxMode + `$`)
parts, err := p.matchesToVolumeSpecParts(spec, specExp)
if err != nil {
return nil, err
}
return newVolume(parts["source"], parts["destination"], parts["mode"]), nil
}
package parser
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestLinuxParser_ParseVolume(t *testing.T) {
testCases := map[string]struct {
volumeSpec string
expectedParts *Volume
expectedError error
}{
"empty": {
volumeSpec: "",
expectedError: NewInvalidVolumeSpecErr(""),
},
"destination only": {
volumeSpec: "/destination",
expectedParts: &Volume{Destination: "/destination"},
},
"source and destination": {
volumeSpec: "/source:/destination",
expectedParts: &Volume{Source: "/source", Destination: "/destination"},
},
"destination and mode": {
volumeSpec: "/destination:rw",
expectedParts: &Volume{Destination: "/destination", Mode: "rw"},
},
"all values": {
volumeSpec: "/source:/destination:rw",
expectedParts: &Volume{Source: "/source", Destination: "/destination", Mode: "rw"},
},
"too much colons": {
volumeSpec: "/source:/destination:rw:something",
expectedError: NewInvalidVolumeSpecErr("/source:/destination:rw:something"),
},
"invalid source": {
volumeSpec: ":/destination",
expectedError: NewInvalidVolumeSpecErr(":/destination"),
},
"named source": {
volumeSpec: "volume_name:/destination",
expectedParts: &Volume{Source: "volume_name", Destination: "/destination"},
},
}
for testName, testCase := range testCases {
t.Run(testName, func(t *testing.T) {
parser := newLinuxParser()