Commit 5f9c9357 authored by Alessio Caiazza's avatar Alessio Caiazza Committed by Kamil Trzciński

Remove go-bindata

parent 342f1fd5
......@@ -4,11 +4,10 @@ config.toml
out
builds/
commands/helpers/archive.zip
executors/docker/bindata.go
dockerfiles/checksums
dockerfiles/alpine/gitlab-runner-linux-amd64
dockerfiles/alpine/checksums
dockerfiles/build/gitlab-runner-helper
dockerfiles/build/binaries/gitlab-runner-helper.*
dockerfiles/ubuntu/gitlab-runner_amd64.deb
dockerfiles/ubuntu/checksums
.idea/
......
......@@ -62,16 +62,14 @@ prepare alpine-no-root image:
# Prebuild stage
#
bindata:
helper images:
<<: *docker
stage: prebuild
script:
- make docker
artifacts:
paths:
- out/docker/prebuilt-x86_64.tar.xz
- out/docker/prebuilt-arm.tar.xz
- executors/docker/bindata.go
- out/helper-images/
- .gopath/bin
- .gopath/.ok
expire_in: 7d
......@@ -265,10 +263,6 @@ binaries windows/386 windows/amd64: *binaries
<<: *except_docs
stage: package
script:
- mkdir -p out/docker
- touch out/docker/prebuilt-x86_64.tar.xz
- touch out/docker/prebuilt-arm.tar.xz
- touch executors/docker/bindata.go
- source ci/touch_make_dependencies
- make ${CI_JOB_NAME}
artifacts:
......
......@@ -57,15 +57,16 @@ export CGO_ENABLED ?= 0
# Development Tools
GOX = $(GOPATH_BIN)/gox
GOBINDATA = $(GOPATH_BIN)/go-bindata
MOCKERY = $(GOPATH_BIN)/mockery
DEVELOPMENT_TOOLS = $(GOX) $(GOBINDATA) $(MOCKERY)
DEVELOPMENT_TOOLS = $(GOX) $(MOCKERY)
MOCKERY_FLAGS = -note="This comment works around https://github.com/vektra/mockery/issues/155"
.PHONY: clean version mocks
all: deps build
all: deps docker build
include Makefile.docker.mk
help:
# Commands:
......@@ -101,78 +102,7 @@ deps: $(DEVELOPMENT_TOOLS)
codequality:
./scripts/codequality analyze --dev
out/docker/prebuilt-x86_64.tar.xz: $(GO_FILES) $(GOX)
# Create directory
mkdir -p out/docker
ifneq (, $(shell docker info))
# Building gitlab-runner-helper
gox -osarch=linux/amd64 \
-ldflags "$(GO_LDFLAGS)" \
-output="dockerfiles/build/gitlab-runner-helper" \
$(PKG)/apps/gitlab-runner-helper
# Build docker images
docker build -t gitlab/gitlab-runner-helper:x86_64-$(REVISION) -f dockerfiles/build/Dockerfile.x86_64 dockerfiles/build
-docker rm -f gitlab-runner-prebuilt-x86_64-$(REVISION)
docker create --name=gitlab-runner-prebuilt-x86_64-$(REVISION) gitlab/gitlab-runner-helper:x86_64-$(REVISION) /bin/sh
docker export -o out/docker/prebuilt-x86_64.tar gitlab-runner-prebuilt-x86_64-$(REVISION)
docker rm -f gitlab-runner-prebuilt-x86_64-$(REVISION)
xz -f -9 out/docker/prebuilt-x86_64.tar
else
$(warning =============================================)
$(warning WARNING: downloading prebuilt docker images that will be embedded in gitlab-runner)
$(warning WARNING: to use images compiled from your code install Docker Engine)
$(warning WARNING: and remove out/docker/prebuilt-x86_64.tar.xz)
$(warning =============================================)
curl -o out/docker/prebuilt-x86_64.tar.xz \
https://gitlab-runner-downloads.s3.amazonaws.com/master/docker/prebuilt-x86_64.tar.xz
endif
out/docker/prebuilt-arm.tar.xz: $(GO_FILES) $(GOX)
# Create directory
mkdir -p out/docker
ifneq (, $(shell docker info))
# Building gitlab-runner-helper
gox -osarch=linux/arm \
-ldflags "$(GO_LDFLAGS)" \
-output="dockerfiles/build/gitlab-runner-helper" \
$(PKG)/apps/gitlab-runner-helper
# Build docker images
docker build -t gitlab/gitlab-runner-helper:arm-$(REVISION) -f dockerfiles/build/Dockerfile.arm dockerfiles/build
-docker rm -f gitlab-runner-prebuilt-arm-$(REVISION)
docker create --name=gitlab-runner-prebuilt-arm-$(REVISION) gitlab/gitlab-runner-helper:arm-$(REVISION) /bin/sh
docker export -o out/docker/prebuilt-arm.tar gitlab-runner-prebuilt-arm-$(REVISION)
docker rm -f gitlab-runner-prebuilt-arm-$(REVISION)
xz -f -9 out/docker/prebuilt-arm.tar
else
$(warning =============================================)
$(warning WARNING: downloading prebuilt docker images that will be embedded in gitlab-runner)
$(warning WARNING: to use images compiled from your code install Docker Engine)
$(warning WARNING: and remove out/docker/prebuilt-arm.tar.xz)
$(warning =============================================)
curl -o out/docker/prebuilt-arm.tar.xz \
https://gitlab-runner-downloads.s3.amazonaws.com/master/docker/prebuilt-arm.tar.xz
endif
executors/docker/bindata.go: out/docker/prebuilt-x86_64.tar.xz out/docker/prebuilt-arm.tar.xz $(GOBINDATA)
# Generating embedded data
go-bindata \
-pkg docker \
-nocompress \
-nomemcopy \
-nometadata \
-prefix out/docker/ \
-o executors/docker/bindata.go \
out/docker/prebuilt-x86_64.tar.xz \
out/docker/prebuilt-arm.tar.xz
go fmt executors/docker/bindata.go
docker: executors/docker/bindata.go
build: executors/docker/bindata.go $(GOX)
build: $(GOX)
# Building $(NAME) in version $(VERSION) for $(BUILD_PLATFORMS)
gox $(BUILD_PLATFORMS) \
-ldflags "$(GO_LDFLAGS)" \
......@@ -186,16 +116,16 @@ build_simple: $(GOPATH_SETUP)
-o "out/binaries/$(NAME)" \
$(PKG)
build_current: executors/docker/bindata.go build_simple
build_current: build_simple
check_race_conditions: executors/docker/bindata.go
check_race_conditions:
@./scripts/check_race_conditions $(OUR_PACKAGES)
test: executors/docker/bindata.go
test: $(PKG_BUILD_DIR) docker
# Running tests...
go test $(OUR_PACKAGES) $(TESTFLAGS)
install: executors/docker/bindata.go
install:
go install --ldflags="$(GO_LDFLAGS)" $(PKG)
dockerfiles:
......@@ -237,7 +167,7 @@ package: package-deps package-prepare package-deb package-rpm
package-deps:
# Installing packaging dependencies...
gem install rake fpm --no-ri --no-rdoc
which fpm 1>/dev/null || gem install rake fpm --no-ri --no-rdoc
package-prepare:
chmod 755 packaging/root/usr/share/gitlab-runner/
......@@ -283,7 +213,8 @@ package-deb-fpm:
--deb-suggests docker-engine \
-a $(PACKAGE_ARCH) \
packaging/root/=/ \
out/binaries/$(NAME)-linux-$(ARCH)=/usr/bin/gitlab-runner
out/binaries/$(NAME)-linux-$(ARCH)=/usr/lib/gitlab-runner/gitlab-runner \
out/helper-images/=/usr/lib/gitlab-runner/helper-images/
package-rpm-fpm:
@mkdir -p out/rpm/
......@@ -308,7 +239,8 @@ package-rpm-fpm:
--depends tar \
-a $(PACKAGE_ARCH) \
packaging/root/=/ \
out/binaries/$(NAME)-linux-$(ARCH)=/usr/bin/gitlab-runner
out/binaries/$(NAME)-linux-$(ARCH)=/usr/lib/gitlab-runner/gitlab-runner \
out/helper-images/=/usr/lib/gitlab-runner/helper-images/
packagecloud: packagecloud-deps packagecloud-deb packagecloud-rpm
......@@ -365,12 +297,11 @@ release_s3: prepare_windows_zip prepare_zoneinfo prepare_index
# Releasing to S3
@./ci/release_s3
prepare_windows_zip:
# Prepare ZIP archives for Windows *.exe files
@cd out/binaries/; \
for file in $$(find * -type f -name "*windows*.exe" | sed 's/.exe$$//'); do \
zip $${file}.zip $${file}.exe; \
done
out/binaries/gitlab-runner-windows-%.zip: out/binaries/gitlab-runner-windows-%.exe
zip --junk-paths $@ $<
cd out/ && zip -r ../$@ helper-images
prepare_windows_zip: out/binaries/gitlab-runner-windows-386.zip out/binaries/gitlab-runner-windows-amd64.zip
prepare_zoneinfo:
# preparing the zoneinfo file
......@@ -422,13 +353,9 @@ $(PKG_BUILD_DIR):
$(GOX): $(GOPATH_SETUP)
go get github.com/mitchellh/gox
$(GOBINDATA): $(GOPATH_SETUP)
go get github.com/jteeuwen/go-bindata/...
$(MOCKERY): $(GOPATH_SETUP)
go get github.com/vektra/mockery/.../
clean:
-$(RM) -rf $(LOCAL_GOPATH)
-$(RM) executors/docker/bindata.go
-$(RM) -rf $(TARGET_DIR)
ifeq (, $(shell docker info 2>/dev/null))
USE_PRECOMPILED_IMAGES ?= 1
endif
docker: out/helper-images/prebuilt-x86_64.tar.xz out/helper-images/prebuilt-arm.tar.xz
ifeq (, $(USE_PRECOMPILED_IMAGES))
HELPER_GO_FILES ?= $(shell find common network vendor -name '*.go')
GO_x86_64_ARCH = amd64
GO_arm_ARCH = arm
dockerfiles/build/binaries/gitlab-runner-helper.%: $(HELPER_GO_FILES) $(GOX)
gox -osarch=linux/$(GO_$*_ARCH) -ldflags "$(GO_LDFLAGS)" -output=$@ $(PKG)/apps/gitlab-runner-helper
out/helper-images/prebuilt-%.tar: dockerfiles/build/binaries/gitlab-runner-helper.%
@mkdir -p $$(dirname $@_)
docker build -t gitlab/gitlab-runner-helper:$*-$(REVISION) -f dockerfiles/build/Dockerfile.$* dockerfiles/build
-docker rm -f gitlab-runner-prebuilt-$*-$(REVISION)
docker create --name=gitlab-runner-prebuilt-$*-$(REVISION) gitlab/gitlab-runner-helper:$*-$(REVISION) /bin/sh
docker export -o $@ gitlab-runner-prebuilt-$*-$(REVISION)
docker rm -f gitlab-runner-prebuilt-$*-$(REVISION)
out/helper-images/prebuilt-%.tar.xz: out/helper-images/prebuilt-%.tar
xz -f -9 $<
else
out/helper-images/prebuilt-%.tar.xz:
$(warning WARNING: downloading prebuilt docker images that will be loaded by gitlab-runner: $@)
@mkdir -p $$(dirname $@_)
curl -o $@ https://gitlab-runner-downloads.s3.amazonaws.com/master/docker/$(shell basename $@)
endif
for file in $(find out/* .gopath/* .gopath/.ok -type f); do
for file in $(find out .gopath -type f); do
touch $file
done
test -f out/docker/prebuilt-x86_64.tar.xz
test -f out/docker/prebuilt-arm.tar.xz
test -f executors/docker/bindata.go && touch executors/docker/bindata.go
......@@ -3,4 +3,5 @@ FROM multiarch/alpine:armhf-v3.7
RUN apk add --update bash ca-certificates git miniperl \
&& ln -s miniperl /usr/bin/perl
COPY ./ /usr/bin
COPY ./scripts/ /usr/bin
COPY ./binaries/gitlab-runner-helper.arm /usr/bin/gitlab-runner-helper
......@@ -5,5 +5,5 @@ RUN apk add --no-cache bash ca-certificates git git-lfs miniperl \
RUN git lfs install
COPY ./ /usr/bin
COPY ./scripts/ /usr/bin
COPY ./binaries/gitlab-runner-helper.x86_64 /usr/bin/gitlab-runner-helper
package docker
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestPrebuiltX86_64Assets(t *testing.T) {
_, err := Asset("prebuilt-x86_64" + prebuiltImageExtension)
assert.NoError(t, err)
}
func TestPrebuiltARMAssets(t *testing.T) {
_, err := Asset("prebuilt-arm" + prebuiltImageExtension)
assert.NoError(t, err)
}
......@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"regexp"
......@@ -20,7 +21,9 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/pkg/stdcopy"
"github.com/kardianos/osext"
"github.com/mattn/go-zglob"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
......@@ -39,6 +42,8 @@ const (
DockerExecutorStagePullingImage common.ExecutorStage = "docker_pulling_image"
)
var DockerPrebuiltImagesPaths []string
var neverRestartPolicy = container.RestartPolicy{Name: "no"}
type executor struct {
......@@ -61,6 +66,18 @@ type executor struct {
usedImagesLock sync.RWMutex
}
func init() {
runnerFolder, err := osext.ExecutableFolder()
if err != nil {
logrus.Errorln("Docker executor: unable to detect gitlab-runner folder, prebuilt image helpers will be loaded from DockerHub.", err)
}
DockerPrebuiltImagesPaths = []string{
filepath.Join(runnerFolder, "helper-images"),
filepath.Join(runnerFolder, "out/helper-images"),
}
}
func (e *executor) getServiceVariables() []string {
return e.Build.GetAllVariables().PublicOrInternal().StringList()
}
......@@ -237,6 +254,38 @@ func (e *executor) getArchitecture() string {
}
}
func (e *executor) loadPrebuiltImage(path, ref, tag string) (*types.ImageInspect, error) {
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
if os.IsNotExist(err) {
return nil, err
}
return nil, fmt.Errorf("Cannot load prebuilt image: %s: %q", path, err.Error())
}
defer file.Close()
e.Debugln("Loading prebuilt image...")
source := types.ImageImportSource{
Source: file,
SourceName: "-",
}
options := types.ImageImportOptions{Tag: tag}
if err := e.client.ImageImportBlocking(e.Context, source, ref, options); err != nil {
return nil, fmt.Errorf("Failed to import image: %s", err)
}
image, _, err := e.client.ImageInspectWithRaw(e.Context, ref+":"+tag)
if err != nil {
e.Debugln("Inspecting imported image", ref, "failed:", err)
return nil, err
}
return &image, err
}
func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
if imageNameFromConfig := e.Config.Docker.HelperImage; imageNameFromConfig != "" {
e.Debugln("Pull configured helper_image for predefined container instead of import bundled image", imageNameFromConfig, "...")
......@@ -248,40 +297,35 @@ func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
return nil, errors.New("unsupported docker architecture")
}
imageName := prebuiltImageName + ":" + architecture + "-" + common.REVISION
revision := "latest"
if common.REVISION != "HEAD" {
revision = common.REVISION
}
// Try to find already loaded prebuilt image
tag := fmt.Sprintf("%s-%s", architecture, revision)
imageName := fmt.Sprintf("%s:%s", prebuiltImageName, tag)
e.Debugln("Looking for prebuilt image", imageName, "...")
image, _, err := e.client.ImageInspectWithRaw(e.Context, imageName)
if err == nil {
return &image, nil
}
data, err := Asset("prebuilt-" + architecture + prebuiltImageExtension)
if err != nil {
return nil, fmt.Errorf("Unsupported architecture: %s: %q", architecture, err.Error())
}
e.Debugln("Loading prebuilt image...")
ref := prebuiltImageName
source := types.ImageImportSource{
Source: bytes.NewBuffer(data),
SourceName: "-",
}
options := types.ImageImportOptions{
Tag: architecture + "-" + common.REVISION,
}
if err := e.client.ImageImportBlocking(e.Context, source, ref, options); err != nil {
return nil, fmt.Errorf("Failed to import image: %s", err)
}
// Try to load prebuilt image from local filesystem
for _, dockerPrebuiltImagesPath := range DockerPrebuiltImagesPaths {
dockerPrebuiltImageFilePath := filepath.Join(dockerPrebuiltImagesPath, "prebuilt-"+architecture+prebuiltImageExtension)
image, err := e.loadPrebuiltImage(dockerPrebuiltImageFilePath, prebuiltImageName, tag)
if err != nil {
e.Debugln("Failed to load prebuilt image from:", dockerPrebuiltImageFilePath, "error:", err)
continue
}
image, _, err = e.client.ImageInspectWithRaw(e.Context, imageName)
if err != nil {
e.Debugln("Inspecting imported image", imageName, "failed:", err)
return nil, err
return image, err
}
return &image, err
// Fallback to getting image from DockerHub
e.Debugln("Loading image from registry:", imageName)
return e.getDockerImage(imageName)
}
func (e *executor) getBuildImage() (*types.ImageInspect, error) {
......@@ -629,7 +673,7 @@ func (e *executor) createService(serviceIndex int, service, version, image strin
serviceSlug := strings.Replace(service, "/", "__", -1)
containerName := fmt.Sprintf("%s-%s-%d", e.Build.ProjectUniqueName(), serviceSlug, serviceIndex)
// this will fail potentially some builds if there'e name collision
// this will fail potentially some builds if there's name collision
e.removeContainer(e.Context, containerName)
config := &container.Config{
......@@ -871,7 +915,7 @@ func (e *executor) createContainer(containerType string, imageDefinition common.
Sysctls: e.Config.Docker.SysCtls,
}
// this will fail potentially some builds if there'e name collision
// this will fail potentially some builds if there's name collision
e.removeContainer(e.Context, containerName)
e.Debugln("Creating container", containerName, "...")
......@@ -1072,7 +1116,7 @@ func (e *executor) verifyAllowedImage(image, optionName string, allowedImages []
return nil
}
e.Println("Please check runner'e configuration: http://doc.gitlab.com/ci/docker/using_docker_images.html#overwrite-image-and-services")
e.Println("Please check runner's configuration: http://doc.gitlab.com/ci/docker/using_docker_images.html#overwrite-image-and-services")
return errors.New("invalid image")
}
......
......@@ -2,6 +2,7 @@ package docker
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"os"
......@@ -25,6 +26,13 @@ import (
"golang.org/x/net/context"
)
func TestMain(m *testing.M) {
DockerPrebuiltImagesPaths = []string{"../../out/helper-images/"}
flag.Parse()
os.Exit(m.Run())
}
// ImagePullOptions contains the RegistryAuth which is inferred from the docker
// configuration for the user, so just mock it out here.
func buildImagePullOptions(e executor, configName string) mock.AnythingOfTypeArgument {
......
../lib/gitlab-runner/gitlab-runner
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment