Commit fea78b8a authored by Alessio Caiazza's avatar Alessio Caiazza

Merge branch '3895-get-os-dependent-pre-build-image-for-docker-executor' into 'master'

Get windows tag for helper image

Closes #3895

See merge request !1239
parents 8ea6bc47 ceaa344b
Pipeline #52983983 passed with stages
in 53 minutes and 47 seconds
......@@ -11,7 +11,6 @@ import (
"path"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
......@@ -29,6 +28,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
docker_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/helperimage"
)
const (
......@@ -233,27 +233,6 @@ func (e *executor) expandAndGetDockerImage(imageName string, allowedImages []str
return image, nil
}
func (e *executor) getArchitecture() string {
architecture := e.info.Architecture
switch architecture {
case "armv6l", "armv7l", "aarch64":
architecture = "arm"
case "amd64":
architecture = "x86_64"
}
if architecture != "" {
return architecture
}
switch runtime.GOARCH {
case "amd64":
return "x86_64"
default:
return runtime.GOARCH
}
}
func (e *executor) loadPrebuiltImage(path, ref, tag string) (*types.ImageInspect, error) {
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
......@@ -298,9 +277,9 @@ func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
return e.getDockerImage(imageNameFromConfig)
}
architecture := e.getArchitecture()
if architecture == "" {
return nil, errors.New("unsupported docker architecture")
helperImageInfo, err := helperimage.GetInfo(e.info)
if err != nil {
return nil, err
}
revision := "latest"
......@@ -308,8 +287,12 @@ func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
revision = common.REVISION
}
tag, err := helperImageInfo.Tag(revision)
if err != nil {
return nil, err
}
// Try to find already loaded prebuilt image
tag := fmt.Sprintf("%s-%s", architecture, revision)
imageName := fmt.Sprintf("%s:%s", prebuiltImageName, tag)
e.Debugln("Looking for prebuilt image", imageName, "...")
image, _, err := e.client.ImageInspectWithRaw(e.Context, imageName)
......@@ -318,6 +301,22 @@ func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
}
// Try to load prebuilt image from local filesystem
loadedImage := e.getLocalDockerImage(helperImageInfo, tag)
if loadedImage != nil {
return loadedImage, nil
}
// Fallback to getting image from DockerHub
e.Debugln("Loading image from registry:", imageName)
return e.getDockerImage(imageName)
}
func (e *executor) getLocalDockerImage(helperImageInfo helperimage.Info, tag string) *types.ImageInspect {
if !helperImageInfo.IsSupportingLocalImport() {
return nil
}
architecture := helperImageInfo.Architecture()
for _, dockerPrebuiltImagesPath := range DockerPrebuiltImagesPaths {
dockerPrebuiltImageFilePath := filepath.Join(dockerPrebuiltImagesPath, "prebuilt-"+architecture+prebuiltImageExtension)
image, err := e.loadPrebuiltImage(dockerPrebuiltImageFilePath, prebuiltImageName, tag)
......@@ -326,12 +325,10 @@ func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
continue
}
return image, err
return image
}
// Fallback to getting image from DockerHub
e.Debugln("Loading image from registry:", imageName)
return e.getDockerImage(imageName)
return nil
}
func (e *executor) getBuildImage() (*types.ImageInspect, error) {
......
......@@ -25,6 +25,7 @@ import (
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/helperimage"
)
func TestMain(m *testing.M) {
......@@ -1354,6 +1355,9 @@ func setUpExecutorForFeatureFlag(variables common.JobVariables, helperImage stri
Context: context.Background(),
},
client: client,
info: types.Info{
OSType: helperimage.OSTypeLinux,
},
}
}
......
package helperimage
import (
"fmt"
"github.com/docker/docker/api/types"
)
const (
OSTypeLinux = "linux"
OSTypeWindows = "windows"
)
// Info provides information about the helper image that can be used to
// pull from Docker Hub.
type Info interface {
Architecture() string
Tag(revision string) (string, error)
IsSupportingLocalImport() bool
}
type unsupportedOSTypeError struct {
detectedOSType string
}
func (e *unsupportedOSTypeError) Error() string {
return fmt.Sprintf("unsupported OSType %q", e.detectedOSType)
}
func newUnsupportedOSTypeError(osType string) *unsupportedOSTypeError {
return &unsupportedOSTypeError{
detectedOSType: osType,
}
}
type infoFactory func(info types.Info) Info
var supportedOsTypesFactories = map[string]infoFactory{
OSTypeWindows: newWindowsInfo,
OSTypeLinux: newLinuxInfo,
}
func GetInfo(info types.Info) (Info, error) {
factory, ok := supportedOsTypesFactories[info.OSType]
if !ok {
return nil, newUnsupportedOSTypeError(info.OSType)
}
return factory(info), nil
}
package helperimage
import (
"testing"
"github.com/docker/docker/api/types"
"github.com/stretchr/testify/assert"
)
func TestGetInfo(t *testing.T) {
testCases := []struct {
osType string
expectedHelperImageType interface{}
expectedError interface{}
}{
{osType: OSTypeLinux, expectedHelperImageType: &linuxInfo{}, expectedError: nil},
{osType: OSTypeWindows, expectedHelperImageType: &windowsInfo{}, expectedError: nil},
{osType: "unsupported", expectedHelperImageType: nil, expectedError: newUnsupportedOSTypeError("unsupported")},
}
for _, testCase := range testCases {
t.Run(testCase.osType, func(t *testing.T) {
i, err := GetInfo(types.Info{OSType: testCase.osType})
assert.IsType(t, testCase.expectedHelperImageType, i)
assert.Equal(t, testCase.expectedError, err)
})
}
}
package helperimage
import (
"fmt"
"runtime"
"github.com/docker/docker/api/types"
)
type linuxInfo struct {
dockerArch string
}
func (u *linuxInfo) Architecture() string {
switch u.dockerArch {
case "armv6l", "armv7l", "aarch64":
return "arm"
case "amd64":
return "x86_64"
}
if u.dockerArch != "" {
return u.dockerArch
}
switch runtime.GOARCH {
case "amd64":
return "x86_64"
default:
return runtime.GOARCH
}
}
func (u *linuxInfo) Tag(revision string) (string, error) {
return fmt.Sprintf("%s-%s", u.Architecture(), revision), nil
}
func (u *linuxInfo) IsSupportingLocalImport() bool {
return true
}
func newLinuxInfo(info types.Info) Info {
return &linuxInfo{
dockerArch: info.Architecture,
}
}
package helperimage
import (
"fmt"
"runtime"
"testing"
"github.com/docker/docker/api/types"
"github.com/stretchr/testify/assert"
)
func Test_linuxInfo_Tag(t *testing.T) {
cases := []struct {
name string
dockerArch string
revision string
expectedTag string
}{
{
name: "When dockerArch not specified we fallback to runtime arch",
dockerArch: "",
revision: "2923a43",
expectedTag: fmt.Sprintf("%s-2923a43", getExpectedArch()),
},
{
name: "Docker runs on armv6l",
dockerArch: "armv6l",
revision: "2923a43",
expectedTag: "arm-2923a43",
},
{
name: "Docker runs on amd64",
dockerArch: "amd64",
revision: "2923a43",
expectedTag: "x86_64-2923a43",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
u := newLinuxInfo(types.Info{Architecture: c.dockerArch})
tag, err := u.Tag(c.revision)
assert.NoError(t, err)
assert.Equal(t, c.expectedTag, tag)
})
}
}
// We re write amd64 to x86_64 for the helper image, and we don't want this test
// to be runtime dependant.
func getExpectedArch() string {
if runtime.GOARCH == "amd64" {
return "x86_64"
}
return runtime.GOARCH
}
func Test_linuxInfo_IsSupportingLocalImport(t *testing.T) {
u := newLinuxInfo(types.Info{})
assert.True(t, u.IsSupportingLocalImport())
}
package helperimage
import (
"errors"
"fmt"
"strings"
"github.com/docker/docker/api/types"
)
const (
windows1809 = "1809"
windows1803 = "1803"
nanoserver1809 = "nanoserver1809"
nanoserver1803 = "nanoserver1803"
windowsSupportedArchitecture = "x86_64"
)
var supportedOSVersions = map[string]string{
windows1803: nanoserver1803,
windows1809: nanoserver1809,
}
var ErrUnsupportedOSVersion = errors.New("could not determine windows version")
type windowsInfo struct {
operatingSystem string
}
func (*windowsInfo) Architecture() string {
return windowsSupportedArchitecture
}
func (u *windowsInfo) Tag(revision string) (string, error) {
osVersion, err := u.osVersion()
if err != nil {
return "", err
}
return fmt.Sprintf("%s-%s-%s", u.Architecture(), revision, osVersion), nil
}
func (u *windowsInfo) osVersion() (string, error) {
for operatingSystem, osVersion := range supportedOSVersions {
if strings.Contains(u.operatingSystem, fmt.Sprintf(" %s ", operatingSystem)) {
return osVersion, nil
}
}
return "", ErrUnsupportedOSVersion
}
func (u *windowsInfo) IsSupportingLocalImport() bool {
return false
}
func newWindowsInfo(info types.Info) Info {
return &windowsInfo{
operatingSystem: info.OperatingSystem,
}
}
package helperimage
import (
"fmt"
"testing"
"github.com/docker/docker/api/types"
"github.com/stretchr/testify/assert"
)
func Test_windowsInfo_Tag(t *testing.T) {
revision := "4011f186"
cases := []struct {
operatingSystem string
expectedVersion string
expectedErr error
}{
{
operatingSystem: "Windows Server 2019 Datacenter Evaluation Version 1809 (OS Build 17763.316)",
expectedVersion: fmt.Sprintf("%s-%s-%s", "x86_64", revision, nanoserver1809),
expectedErr: nil,
},
{
operatingSystem: "Windows Server Datacenter Version 1809 (OS Build 1803.590)",
expectedVersion: fmt.Sprintf("%s-%s-%s", "x86_64", revision, nanoserver1809),
expectedErr: nil,
},
{
operatingSystem: "Windows Server Datacenter Version 1803 (OS Build 17134.590)",
expectedVersion: fmt.Sprintf("%s-%s-%s", "x86_64", revision, nanoserver1803),
expectedErr: nil,
},
{
operatingSystem: "some random string",
expectedErr: ErrUnsupportedOSVersion,
},
}
for _, c := range cases {
t.Run(c.operatingSystem, func(t *testing.T) {
w := newWindowsInfo(types.Info{OperatingSystem: c.operatingSystem})
tag, err := w.Tag(revision)
assert.Equal(t, c.expectedVersion, tag)
assert.Equal(t, c.expectedErr, err)
})
}
}
func Test_windowsInfo_IsSupportingLocalImport(t *testing.T) {
u := newWindowsInfo(types.Info{})
assert.False(t, u.IsSupportingLocalImport())
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment