Commit c0222e53 authored by Grégoire Henry's avatar Grégoire Henry

CI: use the new opam images from `tezos/opam-repository`

parent 8aec39d8
variables:
opam_image: ${CI_REGISTRY_IMAGE}/opam
build_deps_image: ${CI_REGISTRY_IMAGE}/build_deps
public_docker_image: docker.io/${CI_PROJECT_PATH}
## Please update `scripts/version.sh` accordingly
build_deps_image_version: dea9fc16442f36e71caa459f9675928d6b14a800
build_deps_image_name: registry.gitlab.com/tezos/opam-repository
public_docker_image_name: docker.io/${CI_PROJECT_PATH}
stages:
- precheck
- prepare
- build_deps
- build
- test
- opam
- publish
- deploy
- cleanup
## Check zeronet/alphanet
precheck:constant:
image: ${opam_image}
stage: precheck
only:
- zeronet
- alphanet
script:
- ./scripts/check_patch.sh "${CI_COMMIT_REF_NAME}"
## Template for `docker_in_docker` jobs
......@@ -41,48 +26,34 @@ precheck:constant:
tags:
- safe_docker
prepare:opam:
<<: *dind_definition
stage: prepare
only:
variables:
- $UPDATE_OPAM
script:
- ./scripts/ci/create_docker_image.opam.sh "${opam_image}"
- docker push "${opam_image}:alpine-${alpine_version}_ocaml-${ocaml_version}"
- docker tag "${opam_image}:alpine-${alpine_version}_ocaml-${ocaml_version}"
"${opam_image}:latest"
- docker push "${opam_image}:latest"
## Tezos build dependencies
## Tezos
build_deps:
<<: *dind_definition
stage: build_deps
script:
## Building dependencies...
- ./scripts/ci/create_docker_image.build_deps.sh
"${build_deps_image}" "${CI_COMMIT_SHA}"
"${opam_image}:alpine-${alpine_version}_ocaml-${ocaml_version}"
"yes"
.build_deps_template: &build_deps_definition
image: ${build_deps_image}:${CI_COMMIT_SHA}
.build_deps_image_template: &build_deps_image_definition
image: ${build_deps_image_name}:${build_deps_image_version}
except:
- master
- alphanet
- zeronet
- betanet
before_script:
- opam list
- . ./scripts/version.sh
tags:
- gitlab-org
## Tezos
check_deps:
<<: *build_deps_image_definition
stage: build
script:
- if [ "${build_deps_image_version}" != "${opam_repository_tag}" ] ; then
echo "Inconsistent dependencies hash between 'scripts/version.sh' and '.gitlab-ci.yml'." ;
exit 1 ;
fi
- ./scripts/opam-check.sh
build:
<<: *build_deps_definition
<<: *build_deps_image_definition
stage: build
except:
- master
- alphanet
- zeronet
- betanet
script:
- make all
- make build-test
......@@ -94,13 +65,8 @@ build:
## Basic tests (run on merge requests)
.test_template: &test_definition
<<: *build_deps_definition
<<: *build_deps_image_definition
stage: test
except:
- master
- alphanet
- zeronet
- betanet
dependencies:
- build
......@@ -209,10 +175,17 @@ test:proto:sandbox:
script:
- jbuilder build @runtest_sandbox
test:documentation:
<<: *test_definition
script:
- sudo apk add --no-cache py3-sphinx py3-sphinx_rtd_theme
- sudo ln -s /usr/bin/sphinx-build-3 /usr/bin/sphinx-build
- make doc-html
## Basic tests through opam packaged (run on master only)
.opam_template: &opam_definition
image: ${opam_image}
image: ${build_deps_image_name}:opam--${build_deps_image_version}
stage: opam
dependencies: []
only:
......@@ -556,39 +529,16 @@ publish:docker:minimal:
- zeronet
- betanet
script:
- ./scripts/ci/create_docker_image.build.sh
"tezos_build" "${CI_COMMIT_REF_NAME}"
"${build_deps_image}:${CI_COMMIT_SHA}"
- ./scripts/ci/create_docker_image.minimal.sh
- ./scripts/create_docker_image.sh
"${public_docker_image}" "${CI_COMMIT_REF_NAME}"
"tezos_build:${CI_COMMIT_REF_NAME}"
- mkdir ~/.docker || true
- echo "${CI_DOCKER_AUTH}" > ~/.docker/config.json ;
- docker push "${public_docker_image}:${CI_COMMIT_REF_NAME}"
## Publish to github...
publish:github:
image: ${opam_image}
stage: publish
dependencies: []
only:
- master@tezos/tezos
- alphanet@tezos/tezos
script:
- echo "${CI_KH}" > ~/.ssh/known_hosts
- echo "${CI_PK_GITHUB}" > ~/.ssh/id_rsa
- chmod 400 ~/.ssh/id_rsa
- rm -fr .git/refs/original
- sudo apk add --no-cache openssh
- git push git@github.com:tezos/tezos.git -f HEAD:${CI_COMMIT_REF_NAME}
tags:
- gitlab-org
## Publish docs to gitlab...
pages:
image: ${build_deps_image}:${CI_COMMIT_SHA}
image: ${build_deps_image_name}:${build_deps_image_version}
stage: publish
only:
- master@tezos/tezos
......@@ -598,110 +548,6 @@ pages:
script:
- sudo apk add --no-cache py3-sphinx py3-sphinx_rtd_theme
- sudo ln -s /usr/bin/sphinx-build-3 /usr/bin/sphinx-build
- opam install --yes odoc
- make doc-html && sudo mv docs/_build "${CI_PROJECT_DIR}"/public
tags:
- gitlab-org
documentation:
image: ${build_deps_image}:${CI_COMMIT_SHA}
stage: test
only:
- /^.*doc.*$/
script:
- sudo apk add --no-cache py3-sphinx py3-sphinx_rtd_theme
- sudo ln -s /usr/bin/sphinx-build-3 /usr/bin/sphinx-build
- opam install --yes odoc
- make doc-html
tags:
- gitlab-org
## Relaunching the bootstrap servers of the alphanet/zeronet
.bootstrap_template: &bootstrap_definition
image: ${opam_image}
stage: deploy
dependencies: []
before_script:
- echo "${CI_KH}" > ~/.ssh/known_hosts
- echo "${CI_PK_ALPHANET}" | tr -d "\r" > ~/.ssh/id_ed25519 ;
- echo "${CI_SSH_CONFIG}" | tr -d "\r" > ~/.ssh/config
- chmod 600 ~/.ssh/id_ed25519
- sudo apk add --no-cache openssh
allow_failure: true
tags:
- gitlab-org
deploy:bootstrap1:
<<: *bootstrap_definition
only:
- alphanet@tezos/tezos
script:
- ssh bootstrap1
deploy:bootstrap2:
<<: *bootstrap_definition
only:
- alphanet@tezos/tezos
script:
- ssh bootstrap2
deploy:bootstrap3:
<<: *bootstrap_definition
only:
- alphanet@tezos/tezos
script:
- ssh bootstrap3
deploy:bootstrap4:
<<: *bootstrap_definition
only:
- alphanet@tezos/tezos
script:
- ssh bootstrap4
deploy:bootstrap5:
<<: *bootstrap_definition
only:
- alphanet@tezos/tezos
script:
- ssh bootstrap5
deploy:bootstrap6:
<<: *bootstrap_definition
only:
- alphanet@tezos/tezos
script:
- ssh bootstrap6
deploy:zeronet1:
<<: *bootstrap_definition
only:
- zeronet@tezos/tezos
script:
- ssh zeronet1
deploy:zeronet2:
<<: *bootstrap_definition
only:
- zeronet@tezos/tezos
script:
- ssh zeronet2
deploy:zeronet3:
<<: *bootstrap_definition
only:
- zeronet@tezos/tezos
script:
- ssh zeronet3
# cleanup:
# <<: *dind_definition
# stage: cleanup
# dependencies: []
# script:
# - ./scripts/ci/docker_registry_delete.sh
# "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}"
# "${build_deps_image}" "${CI_COMMIT_SHA}"
......@@ -2,18 +2,19 @@
DEV ?= --dev
PACKAGES:=$(patsubst %.opam,%,$(notdir $(shell find src vendors -name \*.opam -print)))
current_ocaml_version := $(shell ocamlc -version)
current_opam_version := $(shell opam --version)
include scripts/version.sh
ifneq (${current_ocaml_version},${ocaml_version})
$(error Unexpected ocaml version (found: ${current_ocaml_version}, expected: ${ocaml_version}))
endif
ifneq (${current_opam_version},${opam_version})
$(error Unexpected opam version (found: ${current_opam_version}, expected: ${opam_version}))
endif
current_ocaml_version := $(shell opam exec -- ocamlc -version)
all:
ifneq (${current_ocaml_version},${ocaml_version})
$(error Unexpected ocaml version (found: ${current_ocaml_version}, expected: ${ocaml_version}))
endif
@jbuilder build ${DEV} \
src/bin_node/main.exe \
src/bin_client/main_client.exe \
......
#!/bin/sh
set -e
ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
script_dir="$(dirname "$ci_dir")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. "$script_dir"/version.sh
build_dir=${build_dir:-_docker_build}
if [ -f "$build_dir"/hidapi-$hidapi_version-r0.apk ] && \
[ -f "$build_dir"/hidapi-dev-$hidapi_version-r0.apk ] && \
[ -d "$build_dir"/keys/ ] ; then
exit 0
fi
tmp_image="hidapi.apk-builder"
tmp_dir=$(mktemp -dt tezos.hidapi.XXXXXXXX)
cleanup () {
set +e
rm -rf "$tmp_dir"
if ! [ -z "$container" ]; then docker rm $container; fi
# docker rmi $tmp_image || true
}
trap cleanup EXIT INT
cp -a "$script_dir"/ci/hidapi-$hidapi_version.APKBUILD "$tmp_dir"/APKBUILD
cat <<EOF > "$tmp_dir/Dockerfile"
FROM andyshinn/alpine-abuild:v4
ENV PACKAGER "Tezos <ci@tezos.com>"
WORKDIR /home/builder/
RUN abuild-keygen -a -i
COPY APKBUILD .
RUN abuilder -r
EOF
echo
echo "### Building hidapi..."
echo
docker build -t $tmp_image "$tmp_dir"
mkdir -p "$build_dir"
container=$(docker create $tmp_image)
docker cp -L $container:/etc/apk/keys "$build_dir"
docker cp -L $container:/packages/home/x86_64/hidapi-$hidapi_version-r0.apk \
"$build_dir"
docker cp -L $container:/packages/home/x86_64/hidapi-dev-$hidapi_version-r0.apk \
"$build_dir"
#! /bin/sh
set -e
ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
script_dir="$(dirname "$ci_dir")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. "$script_dir"/version.sh
build_dir=${build_dir:-_docker_build}
if [ -f "$build_dir"/opam-$opam_tag ] && \
[ -f "$build_dir"/opam-installer-$opam_tag ] ; then
exit 0
fi
tmp_image="opam.bin-builder"
tmp_dir=$(mktemp -dt tezos.opam.XXXXXXXX)
cleanup () {
set +e
rm -rf "$tmp_dir"
if ! [ -z "$container" ]; then docker rm $container; fi
# docker rmi $tmp_image || true
}
trap cleanup EXIT INT
opam_tag="${1:-$opam_tag}"
if ! [ -f "$build_dir"/opam-$opam_tag.tgz ]; then
echo
echo "### Fetching opam-$opam_tag.tar.gz ..."
echo
mkdir -p "$build_dir"
wget -O "$build_dir"/opam-$opam_tag.tgz \
https://github.com/ocaml/opam/archive/$opam_tag.tar.gz
fi
echo
echo "### Building opam ..."
echo
cp -a "$build_dir"/opam-$opam_tag.tgz "$tmp_dir"
cat <<EOF > "$tmp_dir"/Dockerfile
FROM ocaml/opam:alpine-3.6_ocaml-4.06.0
ENV PACKAGER "Tezos <ci@tezos.com>"
COPY opam-$opam_tag.tgz opam-$opam_tag.tgz
RUN tar xzf opam-$opam_tag.tgz && \
cd opam-$opam_tag && \
./configure && \
make lib-ext && \
make
EOF
docker build -t $tmp_image "$tmp_dir"
container=$(docker create $tmp_image)
docker cp -L $container:/home/opam/opam-$opam_tag/opam "$build_dir"/opam-$opam_tag
docker cp -L $container:/home/opam/opam-$opam_tag/opam-installer "$build_dir"/opam-installer-$opam_tag
......@@ -14,28 +14,32 @@ tmp_dir=$(mktemp -dt tezos.opam.tezos.XXXXXXXX)
cleanup () {
set +e
echo Cleaning up...
rm -rf "$tmp_dir"
rm -rf Dockerfile
}
trap cleanup EXIT INT
image_name="${1:-tezos_build}"
image_version="${2:-latest}"
base_image_name="${3-${image_name}_deps:${image_version}}"
cat <<EOF > "$src_dir"/Dockerfile
FROM $base_image_name
COPY Makefile tezos/
COPY src tezos/src/
COPY vendors tezos/vendors/
RUN sudo chown -R opam tezos && \
cd tezos && opam exec -- make all build-test
base_image="${3-${image_name}_deps:${image_version}}"
mkdir -p "$tmp_dir"/tezos/scripts
cp -a Makefile "$tmp_dir"/tezos
cp -a scripts/version.sh "$tmp_dir"/tezos/scripts/
cp -a src "$tmp_dir"/tezos
cp -a vendors "$tmp_dir"/tezos
cat <<EOF > "$tmp_dir"/Dockerfile
FROM $base_image
COPY --chown=tezos:nogroup tezos tezos
RUN opam exec -- make -C tezos all build-test
EOF
echo
echo "### Building tezos..."
echo
docker build -t "$image_name:$image_version" "$src_dir"
docker build -t "$image_name:$image_version" "$tmp_dir"
echo
echo "### Succesfully build docker image: $image_name:$image_version"
......
#! /bin/sh
set -e
ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
script_dir="$(dirname "$ci_dir")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. "$script_dir"/version.sh
tmp_dir=$(mktemp -dt tezos.build_deps.tezos.XXXXXXXX)
cleanup () {
set +e
echo Cleaning up...
rm -rf "$tmp_dir"
}
trap cleanup EXIT INT
opam_files=$(find -name \*.opam | sort)
dependencies="$opam_files scripts/install_build_deps.raw.sh scripts/version.sh scripts/opam-pin.sh scripts/opam-unpin.sh scripts/opam-remove.sh"
image_name="${1:-tezos_build_deps}"
image_version="${2:-latest}"
base_image="${3:-tezos_opam:alpine-${alpine_version}_ocaml-${ocaml_version}}"
cached_image="${4:-}"
cat <<EOF > "$tmp_dir"/Dockerfile
FROM $base_image
COPY . tezos
RUN sudo chown -R opam tezos
RUN cd tezos && opam exec -- ./scripts/install_build_deps.raw.sh
EOF
tar -c $dependencies | tar -C "$tmp_dir" -x
## Lookup for for prebuilt dependencies...
if [ ! -z "$cached_image" ]; then
cd "$tmp_dir"
base_image_sha1=$("$ci_dir"/docker_registry_read.sh \
"${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" \
"$base_image")
dependencies_sha1=cache_$(echo "$base_image_sha1" | sha1sum - $dependencies Dockerfile | sha1sum | tr -d ' -')
echo
echo "### Looking for prebuilt dependencies ($image_name:$dependencies_sha1)..."
if "$ci_dir"/docker_registry_tag.sh gitlab-ci-token "${CI_BUILD_TOKEN}" \
"$image_name" "$dependencies_sha1" "$image_version" ; then
echo
echo "### Found $image_name:$dependencies_sha1"
echo
exit 0
fi
echo
echo "### Missing..."
echo
fi
echo
echo "### Building tezos dependencies..."
echo
docker build -t "$image_name:$image_version" "$tmp_dir"
echo
echo "### Succesfully build docker image: $image_name:$image_version"
echo
if [ ! -z "$cached_image" ]; then
echo
echo "### Saving docker image ($image_name:$dependencies_sha1)..."
echo
docker push "$image_name:$image_version"
"$ci_dir"/docker_registry_tag.sh \
"${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" \
"$image_name" "$image_version" "$dependencies_sha1"
fi
......@@ -9,12 +9,12 @@ cd "$src_dir"
. "$script_dir"/version.sh
export build_dir=${build_dir:-_docker_build}
tmp_dir=$(mktemp -dt tezos.opam.tezos.XXXXXXXX)
image_name="${1:-tezos}"
image_version="${2:-latest}"
build_image_name="${3:-${image_name}_build:${image_version}}"
build_image="${3:-registry.gitlab.com/tezos/opam-repository:${opam_repository_tag}}"
base_image="${4-registry.gitlab.com/tezos/opam-repository:minimal--${opam_repository_tag}}"
cleanup () {
set +e
......@@ -24,18 +24,12 @@ cleanup () {
}
trap cleanup EXIT INT
"$ci_dir"/create_apk.hidapi.sh
cp -a "$build_dir"/hidapi-$hidapi_version-r0.apk \
"$build_dir"/keys/ \
"$tmp_dir"
mkdir -p "$tmp_dir"/bin
mkdir -p "$tmp_dir"/scripts
container=$(docker create $build_image_name)
container=$(docker create $build_image)
for bin in tezos-client tezos-admin-client tezos-node \
tezos-alpha-baker tezos-alpha-endorser tezos-alpha-accuser; do
docker cp -L $container:/home/opam/tezos/$bin "$tmp_dir"/bin
docker cp -L $container:/home/tezos/tezos/$bin "$tmp_dir"/bin
done
cp -a "$script_dir"/docker/entrypoint.sh "$tmp_dir"/bin/
cp -a "$script_dir"/docker/entrypoint.inc.sh "$tmp_dir"/bin/
......@@ -48,29 +42,18 @@ echo "### Building minimal docker image..."
echo
cat > "$tmp_dir"/Dockerfile <<EOF
FROM alpine:$alpine_version
FROM $base_image
LABEL distro_style="apk" distro="alpine" distro_long="alpine-$alpine_version" arch="x86_64" operatingsystem="linux"
COPY keys /etc/apk/keys/
COPY hidapi-$hidapi_version-r0.apk .
RUN sudo apk --no-cache add vim
ENV EDITOR=/usr/bin/vi
RUN apk --no-cache add \
libev gmp vim hidapi-$hidapi_version-r0.apk && \
rm hidapi-$hidapi_version-r0.apk
RUN sudo mkdir -p /var/run/tezos/node /var/run/tezos/client && \
sudo chown -R tezos /var/run/tezos
COPY bin/* /usr/local/bin/
COPY scripts/* /usr/local/share/tezos/
RUN adduser -S tezos && \
mkdir -p /var/run/tezos/node /var/run/tezos/client && \
chown -R tezos /var/run/tezos
USER tezos
ENV EDITOR=/usr/bin/vi
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
EOF
......
#! /bin/sh
set -e
ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
script_dir="$(dirname "$ci_dir")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. "$script_dir"/version.sh
export build_dir=${build_dir:-_docker_build}
tmp_dir=$(mktemp -dt tezos.opam.tezos.XXXXXXXX)
cleanup () {
set +e
echo Cleaning up...
rm -rf "$tmp_dir"
}
trap cleanup EXIT INT
"$ci_dir"/create_apk.hidapi.sh
"$ci_dir"/create_binary.opam.sh
"$ci_dir"/create_opam_repository.tezos_deps.sh
echo
echo "### Building base image..."
echo
image_name="${1:-tezos_opam}"
image_version="${2:-alpine-${alpine_version}_ocaml-${ocaml_version}}"
cp -a "$build_dir"/hidapi-$hidapi_version-r0.apk \
"$build_dir"/hidapi-dev-$hidapi_version-r0.apk \
"$build_dir"/keys/ \
"$tmp_dir"
cp "$build_dir"/opam-$opam_tag "$tmp_dir/opam"
cp "$build_dir"/opam-installer-$opam_tag "$tmp_dir/opam-installer"
tar -C "$tmp_dir" -xzf "$build_dir"/opam_repository-tezos_deps-$ocaml_version-$opam_repository_tag.tgz
cat <<EOF > "$tmp_dir"/Dockerfile
FROM alpine:$alpine_version
COPY opam /usr/bin/opam
COPY opam-installer /usr/bin/opam-installer
COPY keys /etc/apk/keys/
COPY hidapi-$hidapi_version-r0.apk .
COPY hidapi-dev-$hidapi_version-r0.apk .
RUN apk --no-cache add \
build-base bash perl xz m4 git curl tar rsync patch sudo jq \
ncurses-dev gmp-dev libev-dev \
pcre-dev zlib-dev \
snappy snappy-dev \
hidapi-$hidapi_version-r0.apk \
hidapi-dev-$hidapi_version-r0.apk && \
adduser -S opam && \
echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \
chmod 440 /etc/sudoers.d/opam && \
chown root:root /etc/sudoers.d/opam && \
sed -i.bak 's/^Defaults.*requiretty//g' /etc/sudoers
USER opam
WORKDIR /home/opam
COPY opam_repository-tezos_deps opam-repository-tezos_deps
RUN mkdir ~/.ssh && \
chmod 700 ~/.ssh && \
git config --global user.email "ci@tezos.com" && \
git config --global user.name "Tezos CI" && \
opam init --disable-sandboxing --bare --no-setup --yes \
tezos_deps /home/opam/opam-repository-tezos_deps && \
opam switch create --yes /home/opam/tezos ocaml-base-compiler.${ocaml_version}
RUN opam install --switch=/home/opam/tezos --yes opam-depext
ENTRYPOINT [ "opam", "exec", "--set-switch", "--switch=/home/opam/tezos", "--" ]
CMD [ "/bin/bash" ]
EOF
docker build --pull -t "$image_name:$image_version" "$tmp_dir"
#! /bin/sh
set -e
ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")"
script_dir="$(dirname "$ci_dir")"
src_dir="$(dirname "$script_dir")"
cd "$src_dir"
. "$script_dir"/version.sh
build_dir=${build_dir:-_docker_build}
if [ -f "$build_dir/opam_repository-tezos_deps-$ocaml_version-$opam_repository_tag.tgz" ]; then
exit 0
fi
tmp_image="opam_bundle-tezos"
tmp_dir=$(mktemp -dt tezos.opam_bundle.XXXXXXXX)
cleanup () {
set +e
rm -rf "$tmp_dir"
if ! [ -z "$container" ]; then docker rm $container; fi
# docker rmi $tmp_image || true
}
trap cleanup EXIT INT