diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index 1361f1fe412aa72ceab07ddf5fdbe877c47bd3d9..d4a9582ded3bb61b60ddab333f2749d261f057bc 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -34,7 +34,7 @@ build-gdk-image: - .base-image-build-buildx - .build-images:rules:build-gdk-image tags: - - high-cpu + - saas-linux-xlarge-amd64 stage: build-images needs: [] script: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 264950ae1cc152b810530603e8bd545274fd9df0..304544468ac3cf5a4c0409111fe1f9145a557ac9 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -906,17 +906,13 @@ .build-images:rules:build-gdk-image: rules: - if: '$QA_RUN_TESTS_ON_GDK !~ /true|yes|1/i' - when: never + when: manual + allow_failure: true - !reference [".qa:rules:package-and-test-never-run", rules] - <<: *if-default-branch-schedule-nightly # already executed in the 2-hourly schedule when: never - - <<: *if-default-branch-refs # Includes scheduled pipelines - variables: - BUILD_GDK_BASE: "true" - # We want to also rebuild the base image if MRs change certain components. + - <<: *if-default-branch-refs - <<: *if-merge-request - variables: - BUILD_GDK_BASE: "true" changes: *gdk-component-patterns # The rest are included to be consistent with .qa:rules:e2e:test-on-gdk - <<: *if-merge-request-targeting-stable-branch diff --git a/qa/gdk/Dockerfile.gdk b/qa/gdk/Dockerfile.gdk index 51e058a7088d6355f833172c3479f74de1a899f2..0926883d00fa817c745ca10d5e8948bdc49715be 100644 --- a/qa/gdk/Dockerfile.gdk +++ b/qa/gdk/Dockerfile.gdk @@ -1,99 +1,66 @@ -# Multi-stage Dockerfile for packaging gdk as executable docker image -# Each stage can be executed in parallel and cached separately based on changes for respective component -# Caches are cleaned for each stage to reduce the footprint of exported cache layers - -ARG BASE_TAG=master - -FROM registry.gitlab.com/gitlab-org/gitlab-build-images/debian-bullseye-slim-ruby-3.1-golang-1.19-rust-1.65-node-16.14-postgresql-13:rubygems-3.4-git-2.36-lfs-2.9-yarn-1.22 as gdk-base - -RUN set -eux; \ - groupadd gdk -g 1000; \ - useradd gdk -m -s /bin/bash -u 1000 -g 1000 - -ENV GEM_HOME=/home/gdk/.gem \ - GEM_PATH=/home/gdk/.gem \ - PATH=$PATH:/home/gdk/.gem/bin - -WORKDIR /home/gdk - -# Reinstall libpcre2 and install postgresql -# See: https://gitlab.com/gitlab-org/gitaly/-/issues/4085 -RUN set -eux; \ - rm -f /usr/lib/libpcre2*; \ - apt-get update && apt-get install -y --reinstall --no-install-recommends \ - libpcre2-16-0 \ - libpcre2-32-0 \ - libpcre2-8-0 \ - libpcre2-dev \ - libpcre2-posix2 \ - && apt-get install -y --no-install-recommends postgresql-13; \ - apt-get autoclean -y - -# Clone GDK and install system dependencies, purge system git -ARG GDK_SHA -ENV GDK_SHA=${GDK_SHA:-main} -RUN set -eux; \ - git -c advice.detachedHead=false clone --depth 1 https://gitlab.com/gitlab-org/gitlab-development-kit.git; \ - git -C gitlab-development-kit fetch --depth 1 origin ${GDK_SHA}; \ - git -C gitlab-development-kit -c advice.detachedHead=false checkout ${GDK_SHA}; \ - mkdir -p gitlab-development-kit/gitlab && chown -R gdk:gdk gitlab-development-kit; \ - apt-get update && apt-get install -y --no-install-recommends $(grep -o '^[^#]*' gitlab-development-kit/packages_debian.txt); \ - apt-get remove -y git git-lfs; \ - apt-get autoclean -y && apt-get autoremove -y +FROM registry.gitlab.com/gitlab-org/gitlab-development-kit/asdf-bootstrapped-verify:main@sha256:14fa752a80df21f840fc48f4be8561bee21b78886ac718652582fdd788d34c32 + +ENV GITLAB_LICENSE_MODE=test \ + GDK_KILL_CONFIRM=true # Allow passwordless /etc/hosts update by gdk user +USER root RUN echo "gdk ALL=(ALL) NOPASSWD: /usr/bin/tee -a /etc/hosts" >> /etc/sudoers USER gdk -WORKDIR /home/gdk/gitlab-development-kit -# Install GDK and gem dependencies -ARG GDK_VERSION -ENV GDK_VERSION=${GDK_VERSION:-0.2.16} -RUN set -eux; \ - gem install gitlab-development-kit -v ${GDK_VERSION} && touch .gitlab-gdk-gem; \ - bundle install; \ - rm -rf ${GEM_HOME}/cache +# Clone GDK at specific sha and bootstrap packages +# +ARG GDK_SHA=747ab64be815f5c239d5d63209527a42bd838e83 +ARG GEM_CACHE=/home/gdk/.asdf/installs/ruby/3.1.4/lib/ruby/gems/3.1.0/cache +RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \ + set -eux; \ + git clone --depth 1 https://gitlab.com/gitlab-org/gitlab-development-kit.git && cd gitlab-development-kit; \ + git fetch --depth 1 origin ${GDK_SHA} && git -c advice.detachedHead=false checkout ${GDK_SHA}; \ + mkdir gitlab && make bootstrap -COPY --chown=gdk:gdk qa/gdk/gdk.yml ./ +WORKDIR /home/gdk/gitlab-development-kit -# Build gitaly -# -COPY --chown=gdk:gdk GITALY_SERVER_VERSION ./gitlab/ -RUN set -eux; \ - make gitaly-setup; \ - rm -rf gitaly/_build/cache \ - gitaly/_build/deps/git/source \ - gitaly/_build/deps/libgit2/source \ - gitaly/_build/deps \ - gitaly/_build/intermediate \ - ${GEM_HOME}/cache \ - && go clean -cache +COPY --chown=gdk:gdk qa/gdk/gdk.yml ./ # Build gitlab-shell # COPY --chown=gdk:gdk GITLAB_SHELL_VERSION ./gitlab/ -RUN set -eux; \ - make gitlab-shell-setup; \ - rm -rf ${GEM_HOME}/cache \ - && go clean -cache +RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \ + set -eux; \ + make gitlab-shell-setup \ + && cd gitlab-shell && go clean -cache -modcache -r # Build gitlab-workhorse # COPY --chown=gdk:gdk VERSION GITLAB_WORKHORSE_VERSION ./gitlab/ COPY --chown=gdk:gdk workhorse ./gitlab/workhorse -RUN set -eux; \ - make gitlab-workhorse-setup \ - && mv gitlab/workhorse ./; \ - rm -rf ${GEM_HOME}/cache \ - && go clean -cache +RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \ + set -eux; \ + make gitlab-workhorse-setup && mv gitlab/workhorse ./ \ + && cd workhorse && go clean -cache -modcache -r + +# Build gitaly +# +COPY --chown=gdk:gdk GITALY_SERVER_VERSION ./gitlab/ +RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \ + set -eux; \ + make gitaly-setup; \ + cd gitaly \ + && go clean -cache -modcache -r \ + && rm -rf _build/cache \ + _build/deps/git/source \ + _build/deps/libgit2/source \ + _build/deps \ + _build/intermediate # Install gitlab gem dependencies # COPY --chown=gdk:gdk Gemfile Gemfile.lock ./gitlab/ COPY --chown=gdk:gdk vendor/gems/ ./gitlab/vendor/gems/ COPY --chown=gdk:gdk gems/ ./gitlab/gems/ -RUN make .gitlab-bundle && rm -rf ${GEM_HOME}/cache +RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \ + make .gitlab-bundle # Install gitlab npm dependencies # @@ -101,13 +68,6 @@ COPY --chown=gdk:gdk package.json yarn.lock ./gitlab/ COPY --chown=gdk:gdk scripts/frontend/postinstall.js ./gitlab/scripts/frontend/postinstall.js RUN make .gitlab-yarn && yarn cache clean -# Executable gdk image -# -FROM registry.gitlab.com/gitlab-org/gitlab/gitlab-qa-gdk-base:${BASE_TAG} as gdk - -ENV GITLAB_LICENSE_MODE=test \ - GDK_KILL_CONFIRM=true - # Copy code COPY --chown=gdk:gdk ./ ./gitlab/ COPY --chown=gdk:gdk qa/gdk/entrypoint ../ @@ -117,18 +77,9 @@ RUN mkdir -p gitlab/tmp/pids \ && rsync -a --remove-source-files workhorse/ gitlab/workhorse/ # Set up GDK -RUN make \ - redis/redis.conf \ - all \ - && gdk kill \ - && rm -rf ${GEM_HOME}/cache \ - gitaly/_build/cache \ - gitaly/_build/deps/git/source \ - gitaly/_build/deps/libgit2/source \ - gitaly/_build/deps \ - gitaly/_build/intermediate \ - && go clean -modcache \ - && go clean -cache +RUN SKIP_WORKHORSE_SETUP=true SKIP_GITLAB_SHELL_SETUP=true SKIP_GITALY_SETUP=true \ + make redis/redis.conf all \ + && gdk kill ENTRYPOINT [ "/home/gdk/entrypoint" ] CMD [ "gdk", "tail" ] diff --git a/qa/gdk/Dockerfile.gdk.dockerignore b/qa/gdk/Dockerfile.gdk.dockerignore index ef1074ed833d7cb902e60e743fb4463da2bae70b..26062339c217a402709ed3ae2860de74b9e38ab8 100644 --- a/qa/gdk/Dockerfile.gdk.dockerignore +++ b/qa/gdk/Dockerfile.gdk.dockerignore @@ -1,4 +1,10 @@ .bundle/ +.gitlab/ +.lefthook/ +.rubocop_todo/ +.vscode/ +builds/ +changelogs/ danger/ doc/ log/*.log @@ -6,6 +12,21 @@ node_modules/ rubocop/ tmp/* +.eslint* +.gitlab-ci.yml +.haml-lint* +.prettier* +.rubocop* +.stylelintrc +.yamllint +docker-compose.yml +Dockerfile.assets +jest* +lefthook.yml +tests.yml + +*.md + db/fixtures/ !db/fixtures/development/01_admin.rb !db/fixtures/development/02_application_settings.rb diff --git a/scripts/build_gdk_image b/scripts/build_gdk_image index 292d315b5ec3c39cf84b5d926d3e5904e2fadd80..8b1fdec2d7d61b2878d96643ab95b24f73295691 100755 --- a/scripts/build_gdk_image +++ b/scripts/build_gdk_image @@ -7,11 +7,12 @@ source "$(dirname "$0")/utils.sh" REGISTRY="${CI_REGISTRY}/${CI_PROJECT_PATH}" SHA_TAG="${CI_COMMIT_SHA}" BRANCH_TAG="${CI_COMMIT_REF_SLUG}" -BASE_TAG=$([ "${BUILD_GDK_BASE}" == "true" ] && echo "${SHA_TAG}" || echo "master") -if [[ -z "${GDK_SHA}" ]]; then - GDK_SHA=$(git ls-remote https://gitlab.com/gitlab-org/gitlab-development-kit.git main | cut -f 1) -fi +# Fetch ruby version based on contents of .ruby-version +RUBY_VERSION=$(cat .ruby-version) +RUBY_VERSION_MINOR=$(echo "$RUBY_VERSION" | awk -F. '{$3=0; OFS="."; print $1,$2,$3}') + +IMAGE="${REGISTRY}/gitlab-qa-gdk" if [[ -n "${CI}" ]]; then OUTPUT_OPTION="--push" @@ -19,34 +20,18 @@ else OUTPUT_OPTION="--load" fi -function build_image() { - local image=$1 - local target=$2 - - echoinfo "Using GDK at SHA ${GDK_SHA}" - - docker buildx build \ - --cache-to="type=inline" \ - --cache-from="${image}:${BRANCH_TAG}" \ - --cache-from="${image}:master" \ - --file="qa/gdk/Dockerfile.gdk" \ - --target="${target}" \ - --platform=${ARCH:-amd64} \ - --tag="${image}:${SHA_TAG}" \ - --tag="${image}:${BRANCH_TAG}" \ - --build-arg="BASE_TAG=${BASE_TAG}" \ - --build-arg="GDK_SHA=${GDK_SHA:-main}" \ - ${OUTPUT_OPTION} \ - . -} - -# Rebuild base image when BUILD_GDK_BASE is set to true -if [[ "${BUILD_GDK_BASE}" == "true" ]]; then - echoinfo "Building GDK base image", "yes" - build_image "${REGISTRY}/gitlab-qa-gdk-base" "gdk-base" -fi - -echoinfo "Building GDK image", "yes" -build_image "${REGISTRY}/gitlab-qa-gdk" "gdk" +echoinfo "Building GDK image with GDK sha: '${GDK_SHA}'" "yes" + +docker buildx build \ + --cache-to="type=inline" \ + --cache-from="${IMAGE}:${BRANCH_TAG}" \ + --cache-from="${IMAGE}:master" \ + --file="qa/gdk/Dockerfile.gdk" \ + --platform=${ARCH:-amd64} \ + --tag="${IMAGE}:${SHA_TAG}" \ + --tag="${IMAGE}:${BRANCH_TAG}" \ + --build-arg="GEM_CACHE=/home/gdk/.asdf/installs/ruby/${RUBY_VERSION}/lib/ruby/gems/${RUBY_VERSION_MINOR}/cache" \ + ${OUTPUT_OPTION} \ + . echosuccess "Built image '${REGISTRY}/gitlab-qa-gdk:${SHA_TAG}'"