Commit 324e955c authored by Daniel Grove's avatar Daniel Grove Committed by Pierre Boutillier

Tooling, CI, Makefile: Dockerfiles and Make

The previous configuration of building with Docker used a lot of
'magic' and required a fair bit of traversal between the shell scripts
to understand what was happening and where. Most of it could be solved
by using Docker Multi-staged Builds but to not constantly break the
cache when just dealing with final artifacts I've split out the builder
Dockerfile from the main Dockerfile. I've also broken out the commands
run into individual make commands while trying to keep the original
shell script API mostly intact.

Running `./scripts/create_docker_image.sh` will work as it originally
did just with the modifications necessary to align the script to also
use the same underlying commands that are in make. I would have removed
the script completely and adjusted the build pipelines, but the Makefile
has an initial requirement of having `opam` installed which might not be
the case for all users or the base build system.

Othewise you can now just `make docker-image` but this will use the make
bindings. It might be a good idea in the future to make your own Docker
"building" image that can just use make within GitLabs CI system, but I
think that was out of scope of this change.
parent 318c9f26
ARG BASE_IMAGE
ARG BASE_IMAGE_VERSION
ARG BASE_IMAGE_VERSION_NON_MIN
ARG BUILD_IMAGE
ARG BUILD_IMAGE_VERSION
FROM ${BUILD_IMAGE}:${BUILD_IMAGE_VERSION} as builder
FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} as intermediate
# Pull in built binaries
COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/tezos-baker-* /home/tezos/bin/
COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/tezos-endorser-* /home/tezos/bin/
COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/tezos-accuser-* /home/tezos/bin/
COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/tezos-client /home/tezos/bin/
COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/tezos-admin-client /home/tezos/bin/
COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/tezos-node /home/tezos/bin/
COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/tezos-signer /home/tezos/bin/
# Add entrypoint scripts
COPY --chown=tezos:nogroup scripts/docker/entrypoint.* /home/tezos/bin/
# Add scripts
COPY --chown=tezos:nogroup scripts/alphanet_version scripts/alphanet.sh src/bin_client/bash-completion.sh active_protocol_versions /home/tezos/scripts/
FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} as debug
RUN sudo apk --no-cache add vim
ENV EDITOR=/usr/bin/vi
COPY --chown=tezos:nogroup --from=intermediate /home/tezos/bin/ /usr/local/bin/
COPY --chown=tezos:nogroup --from=intermediate /home/tezos/scripts/ /usr/local/share/tezos/
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION_NON_MIN} as stripper
COPY --chown=tezos:nogroup --from=intermediate /home/tezos/bin/tezos-* /home/tezos/bin/
RUN strip /home/tezos/bin/tezos*
FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} as bare
COPY --chown=tezos:nogroup --from=stripper /home/tezos/bin/ /usr/local/bin/
COPY --chown=tezos:nogroup --from=intermediate /home/tezos/scripts/ /usr/local/share/tezos
FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} as minimal
COPY --chown=tezos:nogroup --from=stripper /home/tezos/bin/ /usr/local/bin/
COPY --chown=tezos:nogroup --from=intermediate /home/tezos/scripts/ /usr/local/share/tezos
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
......@@ -7,6 +7,18 @@ active_protocol_directories := $(shell tr -- - _ < active_protocol_versions)
current_opam_version := $(shell opam --version)
include scripts/version.sh
DOCKER_IMAGE_NAME := tezos
DOCKER_IMAGE_VERSION := latest
DOCKER_BUILD_IMAGE_NAME := $(DOCKER_IMAGE_NAME)_build
DOCKER_BUILD_IMAGE_VERSION := latest
DOCKER_BARE_IMAGE_NAME := $(DOCKER_IMAGE_NAME)-bare
DOCKER_BARE_IMAGE_VERSION := latest
DOCKER_DEBUG_IMAGE_NAME := $(DOCKER_IMAGE_NAME)-debug
DOCKER_DEBUG_IMAGE_VERSION := latest
DOCKER_DEPS_IMAGE_NAME := registry.gitlab.com/tezos/opam-repository
DOCKER_DEPS_IMAGE_VERSION := ${opam_repository_tag}
DOCKER_DEPS_MINIMAL_IMAGE_VERSION := minimal--${opam_repository_tag}
ifeq ($(filter ${opam_version}.%,${current_opam_version}),)
$(error Unexpected opam version (found: ${current_opam_version}, expected: ${opam_version}.*))
endif
......@@ -122,9 +134,51 @@ build-deps:
build-dev-deps:
@./scripts/install_build_deps.sh --dev
.PHONY: docker-image-build
docker-image-build:
@docker build \
-t $(DOCKER_BUILD_IMAGE_NAME):$(DOCKER_BUILD_IMAGE_VERSION) \
-f build.Dockerfile \
--build-arg BASE_IMAGE=$(DOCKER_DEPS_IMAGE_NAME) \
--build-arg BASE_IMAGE_VERSION=$(DOCKER_DEPS_IMAGE_VERSION) \
.
.PHONY: docker-image-debug
docker-image-debug:
docker build \
-t $(DOCKER_DEBUG_IMAGE_NAME):$(DOCKER_DEBUG_IMAGE_VERSION) \
--build-arg BASE_IMAGE=$(DOCKER_DEPS_IMAGE_NAME) \
--build-arg BASE_IMAGE_VERSION=$(DOCKER_DEPS_MINIMAL_IMAGE_VERSION) \
--build-arg BUILD_IMAGE=$(DOCKER_BUILD_IMAGE_NAME) \
--build-arg BUILD_IMAGE_VERSION=$(DOCKER_BUILD_IMAGE_VERSION) \
--target=debug \
.
.PHONY: docker-image-bare
docker-image-bare:
@docker build \
-t $(DOCKER_BARE_IMAGE_NAME):$(DOCKER_BARE_IMAGE_VERSION) \
--build-arg=BASE_IMAGE=$(DOCKER_DEPS_IMAGE_NAME) \
--build-arg=BASE_IMAGE_VERSION=$(DOCKER_DEPS_MINIMAL_IMAGE_VERSION) \
--build-arg=BASE_IMAGE_VERSION_NON_MIN=$(DOCKER_DEPS_IMAGE_VERSION) \
--build-arg BUILD_IMAGE=$(DOCKER_BUILD_IMAGE_NAME) \
--build-arg BUILD_IMAGE_VERSION=$(DOCKER_BUILD_IMAGE_VERSION) \
--target=bare \
.
.PHONY: docker-image-minimal
docker-image-minimal:
@docker build \
-t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
--build-arg=BASE_IMAGE=$(DOCKER_DEPS_IMAGE_NAME) \
--build-arg=BASE_IMAGE_VERSION=$(DOCKER_DEPS_MINIMAL_IMAGE_VERSION) \
--build-arg=BASE_IMAGE_VERSION_NON_MIN=$(DOCKER_DEPS_IMAGE_VERSION) \
--build-arg BUILD_IMAGE=$(DOCKER_BUILD_IMAGE_NAME) \
--build-arg BUILD_IMAGE_VERSION=$(DOCKER_BUILD_IMAGE_VERSION) \
.
.PHONY: docker-image
docker-image:
@./scripts/create_docker_image.sh
docker-image: docker-image-build docker-image-debug docker-image-bare docker-image-minimal
.PHONY: install
install:
......
ARG BASE_IMAGE=registry.gitlab.com/tezos/opam-repository
ARG BASE_IMAGE_VERSION
FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
WORKDIR /home/tezos
RUN mkdir -p /home/tezos/tezos/scripts
COPY --chown=tezos:nogroup Makefile tezos
COPY --chown=tezos:nogroup active_protocol_versions tezos
COPY --chown=tezos:nogroup scripts/version.sh tezos/scripts/
COPY --chown=tezos:nogroup src tezos/src
COPY --chown=tezos:nogroup vendors tezos/vendors
RUN opam exec -- make -C tezos all build-test
......@@ -9,38 +9,23 @@ cd "$src_dir"
. "$script_dir"/version.sh
tmp_dir=$(mktemp -dt tezos.opam.tezos.XXXXXXXX)
cleanup () {
set +e
echo Cleaning up...
rm -rf "$tmp_dir"
rm -rf Dockerfile
}
trap cleanup EXIT INT
image_name="${1:-tezos_build}"
image_version="${2:-latest}"
base_image="${3-${image_name}_deps:${image_version}}"
mkdir -p "$tmp_dir"/tezos/scripts
cp -a Makefile "$tmp_dir"/tezos
cp -a active_protocol_versions "$tmp_dir"/tezos
cp -a scripts/version.sh "$tmp_dir"/tezos/scripts/
cp -a src "$tmp_dir"/tezos
cp -a vendors "$tmp_dir"/tezos
cat <<EOF > "$tmp_dir"/Dockerfile
FROM $base_image
COPY --chown=tezos:nogroup tezos tezos
RUN opam exec -- make -C tezos all build-test
EOF
base_image="${3-${image_name}_deps}"
base_image_version="${4:-latest}"
echo
echo "### Building tezos..."
echo
docker build -t "$image_name:$image_version" "$tmp_dir"
docker build \
-t "$image_name:$image_version" \
-f build.Dockerfile \
--build-arg="BASE_IMAGE=$base_image" \
--build-arg="BASE_IMAGE_VERSION=${base_image_version}" \
--build-arg="BASE_IMAGE_VERSION=${base_image_version}" \
--build-arg="BUILD_IMAGE_VERSION=${base_image_version}" \
"$src_dir"
echo
echo "### Successfully build docker image: $image_name:$image_version"
......
#! /bin/sh
set -e
set -ex
ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
script_dir="$(dirname "$ci_dir")"
......@@ -9,85 +9,54 @@ cd "$src_dir"
. "$script_dir"/version.sh
tmp_dir=$(mktemp -dt tezos.opam.tezos.XXXXXXXX)
image_name="${1:-tezos}"
image_version="${2:-latest}"
build_image="${3:-registry.gitlab.com/tezos/opam-repository:${opam_repository_tag}}"
base_image="${4-registry.gitlab.com/tezos/opam-repository:minimal--${opam_repository_tag}}"
cleanup () {
set +e
echo Cleaning up...
rm -rf "$tmp_dir"
if ! [ -z "$container" ]; then docker rm $container; fi
}
trap cleanup EXIT INT
mkdir -p "$tmp_dir"/bin
mkdir -p "$tmp_dir"/scripts
container=$(docker create $build_image)
versioned_daemons="$(sed "s/^\(.*\)$/tezos-baker-\1 tezos-endorser-\1 tezos-accuser-\1/g" "active_protocol_versions")"
for bin in tezos-client tezos-admin-client tezos-node $versioned_daemons tezos-signer; do
docker cp -L $container:/home/tezos/tezos/$bin "$tmp_dir"/bin
done
cp -a "$script_dir"/docker/entrypoint.sh "$tmp_dir"/bin/
cp -a "$script_dir"/docker/entrypoint.inc.sh "$tmp_dir"/bin/
cp "$script_dir"/alphanet.sh "$tmp_dir"/scripts/
cp "$script_dir"/alphanet_version "$tmp_dir"/scripts/
cp "$src_dir"/src/bin_client/bash-completion.sh "$tmp_dir"/scripts/
cp "$src_dir"/active_protocol_versions "$tmp_dir"/scripts/
build_image="${3:-registry.gitlab.com/tezos/opam-repository}"
base_image="${4-registry.gitlab.com/tezos/opam-repository}"
base_version="${5-minimal--${opam_repository_tag}}"
echo
echo "### Building minimal docker images..."
echo
cat > "$tmp_dir"/Dockerfile <<EOF
FROM $base_image
RUN sudo apk --no-cache add vim
ENV EDITOR=/usr/bin/vi
COPY bin/* /usr/local/bin/
COPY scripts/* /usr/local/share/tezos/
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
EOF
docker build -t "$image_name-debug:$image_version" "$tmp_dir"
docker build \
-t "$image_name-debug:$image_version" \
--build-arg "BASE_IMAGE=$base_image" \
--build-arg "BASE_IMAGE_VERSION=$base_version" \
--build-arg "BASE_IMAGE_VERSION_NON_MIN=$opam_repository_tag" \
--build-arg="BUILD_IMAGE=${build_image}" \
--build-arg="BUILD_IMAGE_VERSION=${image_version}" \
--target=debug \
"$src_dir"
echo
echo "### Successfully build docker image: $image_name-debug:$image_version"
echo
echo "Stripping tezos binaries from debugging symbols"
strip "$tmp_dir"/bin/tezos*
cat > "$tmp_dir"/Dockerfile <<EOF
FROM $base_image
COPY bin/* /usr/local/bin/
COPY scripts/* /usr/local/share/tezos/
docker build \
-t "$image_name-bare:$image_version" \
--build-arg "BASE_IMAGE=$base_image" \
--build-arg "BASE_IMAGE_VERSION=$base_version" \
--build-arg="BUILD_IMAGE=${build_image}" \
--build-arg="BUILD_IMAGE_VERSION=${image_version}" \
--build-arg "BASE_IMAGE_VERSION_NON_MIN=$opam_repository_tag" \
--target=bare \
"$src_dir"
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
EOF
docker build -t "$image_name:$image_version" "$tmp_dir"
echo
echo "### Successfully build docker image: $image_name:$image_version"
echo "### Successfully build docker image: $image_name-bare:$image_version"
echo
cat > "$tmp_dir"/Dockerfile <<EOF
FROM $base_image
COPY bin/* /usr/local/bin/
COPY scripts/* /usr/local/share/tezos/
EOF
docker build -t "$image_name-bare:$image_version" "$tmp_dir"
docker build \
-t "$image_name:$image_version" \
--build-arg "BASE_IMAGE=$base_image" \
--build-arg "BASE_IMAGE_VERSION=$base_version" \
--build-arg="BUILD_IMAGE=${build_image}" \
--build-arg="BUILD_IMAGE_VERSION=${image_version}" \
--build-arg "BASE_IMAGE_VERSION_NON_MIN=$opam_repository_tag" \
"$src_dir"
echo
echo "### Successfully build docker image: $image_name-bare:$image_version"
echo "### Successfully build docker image: $image_name:$image_version"
echo
......@@ -17,9 +17,9 @@ build_image_name="${image_name}_build"
"$script_dir"/ci/create_docker_image.build.sh \
"$build_image_name" "$image_version" \
"$build_deps_image_name:$build_deps_image_version"
"$build_deps_image_name" "$build_deps_image_version"
"$script_dir"/ci/create_docker_image.minimal.sh \
"$image_name" "$image_version" \
"$build_image_name:$image_version" \
"$build_deps_image_name:minimal--$build_deps_image_version"
"$build_image_name" \
"$build_deps_image_name" "minimal--$build_deps_image_version"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment