Merge branch 'dont-use-dependency-proxy-for-project-access-tokens' into 'master'

ci: Don't use dependency proxy for project access tokens

See merge request !100912



Merged-by: Lin Jen-Shin <jen-shin@gitlab.com>
Approved-by: Alina Mihaila <amihaila@gitlab.com>
Approved-by: Lin Jen-Shin <jen-shin@gitlab.com>
Co-authored-by: Rémy Coutable <remy@rymai.me>

.gitlab-ci.yml

@@ -56,6 +56,11 @@ workflow:
         NOTIFY_PIPELINE_FAILURE_CHANNEL: "f_ruby3"
         OMNIBUS_GITLAB_RUBY3_BUILD: "true"
         OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB_RUBY3"
+    # This work around https://gitlab.com/gitlab-org/gitlab/-/issues/332411 whichs prevents usage of dependency proxy
+    # when pipeline is triggered by a project access token.
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $GITLAB_USER_LOGIN =~ /project_\d+_bot\d*/'
+      variables:
+        GITLAB_DEPENDENCY_PROXY_ADDRESS: ""
     # For `$CI_DEFAULT_BRANCH` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
     # For tags, create a pipeline.
@@ -71,6 +76,9 @@ workflow:
 variables:
   PG_VERSION: "12"
   DEFAULT_CI_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}.patched-golang-${GO_VERSION}-node-16.14-postgresql-${PG_VERSION}:rubygems-3.2-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36"
+  # We set $GITLAB_DEPENDENCY_PROXY to another variable (since it's set at the group level and has higher precedence than .gitlab-ci.yml)
+  # so that we can override $GITLAB_DEPENDENCY_PROXY_ADDRESS in workflow rules.
+  GITLAB_DEPENDENCY_PROXY_ADDRESS: "${GITLAB_DEPENDENCY_PROXY}"
   RAILS_ENV: "test"
   NODE_ENV: "test"
   BUNDLE_WITHOUT: "production:development"

.gitlab/ci/_skip.yml

 # no-op pipeline template for skipping whole child pipeline execution
 
 no-op:
-  image: ${GITLAB_DEPENDENCY_PROXY}alpine:latest
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:latest
   stage: test
   variables:
     GIT_STRATEGY: none

.gitlab/ci/docs.gitlab-ci.yml

@@ -2,7 +2,7 @@
   extends:
     - .default-retry
     - .docs:rules:review-docs
-  image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine
   stage: review
   needs: []
   variables:

.gitlab/ci/frontend.gitlab-ci.yml

@@ -301,17 +301,17 @@ coverage-frontend:
 
 qa-frontend-node:14:
   extends: .qa-frontend-node
-  image: ${GITLAB_DEPENDENCY_PROXY}node:14
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:14
 
 qa-frontend-node:16:
   extends: .qa-frontend-node
-  image: ${GITLAB_DEPENDENCY_PROXY}node:16
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:16
 
 qa-frontend-node:latest:
   extends:
     - .qa-frontend-node
     - .frontend:rules:qa-frontend-node-latest
-  image: ${GITLAB_DEPENDENCY_PROXY}node:latest
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:latest
 
 webpack-dev-server:
   extends:

.gitlab/ci/global.gitlab-ci.yml

@@ -342,7 +342,7 @@
     FOSS_ONLY: '1'
 
 .use-docker-in-docker:
-  image: ${GITLAB_DEPENDENCY_PROXY}docker:${DOCKER_VERSION}
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}docker:${DOCKER_VERSION}
   services:
     - docker:${DOCKER_VERSION}-dind
   variables:

.gitlab/ci/notify.gitlab-ci.yml

 .notify-slack:
-  image: ${GITLAB_DEPENDENCY_PROXY}alpine
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine
   stage: notify
   dependencies: []
   cache: {}

.gitlab/ci/releases.gitlab-ci.yml

@@ -4,7 +4,7 @@
 .merge-train-sync:
   # We don't need/want any global before/after commands, so we overwrite these
   # settings.
-  image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge
   stage: sync
   before_script:
     - apk add --no-cache --update curl bash jq

.gitlab/ci/review-apps/main.gitlab-ci.yml

@@ -32,7 +32,7 @@ review-build-cng-env:
   extends:
     - .default-retry
     - .review:rules:review-build-cng
-  image: ${GITLAB_DEPENDENCY_PROXY}ruby:3.0-alpine3.13
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:3.0-alpine3.13
   stage: prepare
   needs: []
   before_script:

.gitlab/ci/setup.gitlab-ci.yml

@@ -30,7 +30,7 @@ cache gems:
 .absolutely-minimal-job:
   extends:
     - .minimal-job
-  image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge
   variables:
     GIT_STRATEGY: none
 
@@ -79,7 +79,7 @@ verify-ruby-2.7:
 verify-tests-yml:
   extends:
     - .setup:rules:verify-tests-yml
-  image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine3.13
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine3.13
   stage: test
   needs: []
   script:
@@ -116,7 +116,7 @@ generate-frontend-fixtures-mapping:
 
 detect-tests:
   extends: .rails:rules:detect-tests
-  image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
   needs: []
   stage: prepare
   variables:

.gitlab/ci/test-metadata.gitlab-ci.yml

 .tests-metadata-state:
-  image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
   before_script:
     - source scripts/utils.sh
   artifacts:

.gitlab/ci/workhorse.gitlab-ci.yml

 workhorse:verify:
   extends: .workhorse:rules:workhorse
-  image: ${GITLAB_DEPENDENCY_PROXY}golang:${GO_VERSION}
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}golang:${GO_VERSION}
   stage: test
   needs: []
   script:

doc/development/pipelines.md

@@ -587,8 +587,9 @@ The current stages are:
 ### Dependency Proxy
 
 Some of the jobs are using images from Docker Hub, where we also use
-`${GITLAB_DEPENDENCY_PROXY}` as a prefix to the image path, so that we pull
+`${GITLAB_DEPENDENCY_PROXY_ADDRESS}` as a prefix to the image path, so that we pull
 images from our [Dependency Proxy](../user/packages/dependency_proxy/index.md).
+By default, this variable is set from the value of `${GITLAB_DEPENDENCY_PROXY}`.
 
 `${GITLAB_DEPENDENCY_PROXY}` is a group CI/CD variable defined in
 [`gitlab-org`](https://gitlab.com/gitlab-org) as
@@ -596,13 +597,32 @@ images from our [Dependency Proxy](../user/packages/dependency_proxy/index.md).
 defined as:
 
 ```yaml
-image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge
+image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge
 ```
 
 Projects in the `gitlab-org` group pull from the Dependency Proxy, while
 forks that reside on any other personal namespaces or groups fall back to
 Docker Hub unless `${GITLAB_DEPENDENCY_PROXY}` is also defined there.
 
+#### Work around for when a pipeline is started by a Project access token user
+
+When a pipeline is started by a Project access token user (e.g. the `release-tools approver bot` user which
+automatically updates the Gitaly version used in the main project),
+[the Dependency proxy isn't accessible](https://gitlab.com/gitlab-org/gitlab/-/issues/332411#note_1130388163)
+and the job fails at the `Preparing the "docker+machine" executor` step.
+To work around that, we have a special workflow rule, that overrides the
+`${GITLAB_DEPENDENCY_PROXY_ADDRESS}` variable so that Depdendency proxy isn't used in that case:
+
+```yaml
+- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $GITLAB_USER_LOGIN =~ /project_\d+_bot\d*/'
+  variables:
+    GITLAB_DEPENDENCY_PROXY_ADDRESS: ""
+```
+
+NOTE:
+We don't directly override the `${GITLAB_DEPENDENCY_PROXY}` variable because group-level
+variables have higher precedence over `.gitlab-ci.yml` variables.
+
 ### Common job definitions
 
 Most of the jobs [extend from a few CI definitions](../ci/yaml/index.md#extends)