.gitlab-ci-asdf-versions.yml

@@ -2,4 +2,4 @@
 variables:
     GL_ASDF_PRE_COMMIT_VERSION: 3.8.0
     GL_ASDF_SHELLCHECK_VERSION: 0.10.0
-    GL_ASDF_SHFMT_VERSION: 3.8.0
+    GL_ASDF_SHFMT_VERSION: 3.9.0

.gitlab-ci-default-asdf-versions.yml

@@ -2,4 +2,4 @@
 variables:
     GL_COMMON_CI_TASKS_DEFAULT_ASDF_PRE_COMMIT_VERSION: 3.8.0
     GL_COMMON_CI_TASKS_DEFAULT_ASDF_SHELLCHECK_VERSION: 0.10.0
-    GL_COMMON_CI_TASKS_DEFAULT_ASDF_SHFMT_VERSION: 3.8.0
+    GL_COMMON_CI_TASKS_DEFAULT_ASDF_SHFMT_VERSION: 3.9.0

.gitlab-ci.yml

@@ -12,7 +12,7 @@ variables:
 include:
   - local: .gitlab-ci-asdf-versions.yml
   - local: templates/standard.yml
-  - local: kaniko.yml
+  - local: docker.yml
 
   # Tests which run against the downstream pipeline in the
   # gitlab-com/gl-infra/common-ci-task-tests project
@@ -36,25 +36,16 @@ include:
       - IMAGE_SUFFIX: asdf
       - IMAGE_SUFFIX: mise
   variables:
-    KANIKO_BUILD_FILE: Dockerfile.${IMAGE_SUFFIX}
-    KANIKO_EXTRA_ARGS: |
-      --build-arg GL_COMMON_RENOVATE_GIT_VERSION
-      --build-arg GL_COMMON_RENOVATE_GOLANG_VERSION
-      --build-arg GL_COMMON_RENOVATE_JB_VERSION
-      --build-arg GL_COMMON_RENOVATE_JSONNET_TOOL_VERSION
-      --build-arg GL_COMMON_RENOVATE_NODEJS_VERSION
-      --build-arg GL_COMMON_RENOVATE_RUBY_VERSION
-      --build-arg GL_COMMON_RENOVATE_YARN_VERSION
-      --build-arg GL_COMMON_RENOVATE_YQ_VERSION
-      --build-arg GL_COMMON_RENOVATE_PYTHON_VERSION
+    DOCKER_BUILD_FILE: Dockerfile.${IMAGE_SUFFIX}
+    GL_VERSION_YML_FILES: .gitlab-ci-asdf-versions.yml .gitlab-ci-other-versions.yml .gitlab-ci-default-asdf-versions.yml
 
 container_image_builds:
   stage: validate_builds
   variables:
-    KANIKO_DESTINATION: $CI_REGISTRY_IMAGE/${IMAGE_SUFFIX}:${CI_COMMIT_REF_SLUG}
+    DOCKER_DESTINATION: $CI_REGISTRY_IMAGE/${IMAGE_SUFFIX}:${CI_COMMIT_REF_SLUG}
   extends:
+    - .docker_buildx_base
     - .container_builds
-    - .kaniko_base
   rules:
     - if: '$CI_PIPELINE_SOURCE == "scheduled"'
       when: "never"
@@ -80,9 +71,9 @@ container_image_test:
 container_image_tagged:
   stage: release
   variables:
-    KANIKO_DESTINATION: $CI_REGISTRY_IMAGE/${IMAGE_SUFFIX}:${CI_COMMIT_TAG}
+    DOCKER_DESTINATION: $CI_REGISTRY_IMAGE/${IMAGE_SUFFIX}:${CI_COMMIT_TAG}
   extends:
+    - .docker_buildx_base
     - .container_builds
-    - .kaniko_base
   rules:
     - if: "$CI_COMMIT_TAG"

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
       - id: conventional-pre-commit
         stages: [commit-msg]
   - repo: https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks
-    rev: v2.25.3  # renovate:managed:self
+    rev: v2.27.0  # renovate:managed:self
     hooks:
       - id: shellcheck
       - id: shfmt

.tool-versions

@@ -4,4 +4,4 @@
 pre-commit 3.8.0 # datasource=github-releases depName=pre-commit/pre-commit
 python system
 shellcheck 0.10.0 # datasource=github-releases depName=koalaman/shellcheck
-shfmt 3.8.0 # datasource=github-releases depName=mvdan/sh
+shfmt 3.9.0 # datasource=github-releases depName=mvdan/sh

Dockerfile.asdf

 # This image is used by asdf to validate asdf setup scripts in asdf-tool-versions.yml
+# This docker image is deprecated. Use the mise image instead.
+
 FROM ghcr.io/containerbase/base:11.11.0
 ARG GL_COMMON_RENOVATE_GIT_VERSION
 ARG GL_COMMON_RENOVATE_PYTHON_VERSION

appsec-container-scan.md

+# Appsec Container Scanner
+
+## [`.appsec_container_scan`](./kaniko.yml)
+
+This task can be used to trigger a pipeline for scanning container images and include the container scanning results in the project that is triggering this scan.
+The pipeline is triggered in the [`container-scanners`](https://gitlab.com/gitlab-com/gl-security/appsec/container-scanners/) project which is used, maintained by the GitLab Appsec team.
+
+`CONTAINER_SCAN_PIPELINE_TRIGGER_TOKEN` and `CONTAINER_SCAN_PROJECT_API_TOKEN` CI/CD variables need to be configured on the project that is triggering this scan.
+See [this section](https://gitlab.com/gitlab-com/gl-security/appsec/container-scanners/-/tree/master/#how-to-trigger-a-new-container-scan-and-include-results-in-the-target-project) for more information.
+These variables should already be available to projects under `gl-infra` subgroup as they were added as Group CI/CD variables.
+
+```yaml
+container_image_scan:
+  stage: validate
+  variables:
+    IMAGES: "Set this to be the names of the images you wish to scan, comma delimited"
+  needs:
+    - container_image_build
+  extends:
+    - .appsec_container_scan
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE != "schedule"'
+```

appsec-container-scan.yml

+# Note: this is not a full template, just the basis for appsec container scanning
+
+include:
+  - local: .gitlab-ci-other-versions.yml
+
+.appsec_container_scan:
+  image: registry.gitlab.com/gitlab-com/gl-security/appsec/container-scanners:latest
+  environment:
+    name: appsec_container_scanning
+    action: prepare
+  script:
+    - BUILD_WORKING_DIRECTORY="$PWD"
+    - cd /run
+    - ./appsec-container-scan "${IMAGES}" > $BUILD_WORKING_DIRECTORY/gl-container-scanning-report.json
+    - cd $BUILD_WORKING_DIRECTORY
+  allow_failure: true
+  artifacts:
+    reports:
+      container_scanning: gl-container-scanning-report.json

asdf-tool-versions.md

@@ -31,6 +31,6 @@ include:
   # and that asdf and mise are generally working
   # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/asdf-tool-versions.md
   - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: asdf-tool-versions.yml
 ```

asdf-tool-versions.yml

@@ -6,7 +6,7 @@ spec:
 validate_mise_tool_versions:
   stage: $[[ inputs.stage ]]
   image:
-    name: registry.gitlab.com/gitlab-com/gl-infra/common-ci-tasks/mise:v2.26.0
+    name: registry.gitlab.com/gitlab-com/gl-infra/common-ci-tasks/mise:v2.27.0
     entrypoint: [""]
   needs: []
   script:

checkov.md

@@ -25,6 +25,6 @@ include:
   # Runs checkov on all terraform module directories
   # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/checkov.md
   - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: checkov.yml
 ```

container-diff.md

@@ -7,7 +7,7 @@ This can help to determine how much a container image has changed in size due to
 ```yaml
 include:
   - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: 'container-diff.yml'
     inputs:
       job_name: container-diff # The name of the job this template will create

danger.md

@@ -13,7 +13,7 @@ variables:
 include:
   # Run Danger during merge requests to alert on messages, warnings and errors.
   - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: 'danger.yml'
     inputs:
         # stage: defaults to `validate`

docker.md

+# Docker Extensible Tasks
+
+## [`.docker_buildx_base`](./docker.yml)
+
+This is not a full template, but allows users to easily define their own [docker buildx](https://docs.docker.com/reference/cli/docker/buildx/build/) container image builds without all the boilerplate.
+
+The following variables can be configured:
+
+* `DOCKER_DESTINATION`: **Required**. The destination to push the tag to.
+* `DOCKER_BUILD_FILE`: The Dockerfile to build. Defaults to `Dockerfile`.
+* `DOCKER_BUILD_CONTEXT`: The context to use. Defaults to `.`.
+* `DOCKER_BUILDX_EXTRA_ARGS`: Additional arguments to add to the `docker buildx build` command.
+* `GL_VERSION_YML_FILES`: A set of `.gitlab-ci.yml` include files containing `variables`.
+   These variables will be passed through as `--build-args`.
+   Defaults to `.gitlab-ci-asdf-versions.yml .gitlab-ci-other-versions.yml`.
+
+### Usage Example
+
+```yaml
+include:
+  # Includes a base template for running an opinionated docker buildx build
+  # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docker.md
+  - project: 'gitlab-com/gl-infra/common-ci-tasks'
+    ref: v2.27.0  # renovate:managed
+    file: 'docker.yml'
+
+.container_builds:
+  stage: release
+  variables:
+    # DOCKER_BUILD_FILE: the Dockerfile to build
+    DOCKER_BUILD_FILE: Dockerfile.alt # Defaults to Dockerfile
+    # DOCKER_BUILD_CONTEXT: the build context to use
+    DOCKER_BUILD_CONTEXT: sub/directory/ # Defaults to project root
+    # DOCKER_BUILDX_EXTRA_ARGS: additional arguments for docker buildx build
+    DOCKER_BUILDX_EXTRA_ARGS: |
+      --build-arg GL_DEDICATED_CONTAINER_IMAGE_VERSION_PREFIXED
+  extends:
+    - .docker_buildx_base
+
+container_image_build:
+  variables:
+    DOCKER_DESTINATION: $CI_REGISTRY_IMAGE/onboard:${CI_COMMIT_REF_SLUG}
+  extends:
+    - .container_builds
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE != "schedule"'
+
+# Tags use the git tag, not the slug
+container_image_tagged:
+  variables:
+    DOCKER_DESTINATION: $CI_REGISTRY_IMAGE/onboard:${CI_COMMIT_TAG}
+  extends:
+    - .container_builds
+  rules:
+    - if: '$CI_COMMIT_TAG'
+```
+
+### Signing and Verification
+
+The `.docker_buildx_base` task signs container images using keyless signatures based on OIDC tokens.
+
+The [Sigstore project](https://www.sigstore.dev/)  provides a CLI called
+[Cosign](https://docs.sigstore.dev/signing/quickstart/) which is used for keyless
+signing of container images built with GitLab CI/CD.
+
+This allows certificates to be verified without the need for managing secret keys.
+
+Following the signing of the OIDC-based certificate identity that was used to sign the container is displayed in
+logs:
+
+```
+------------------------------------------------------------
+Verify this container image using:
+cosign verify registry.gitlab.com/gitlab-com/gl-infra/common-ci-tasks/asdf:v2.27.0 \
+  --certificate-identity https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks//.gitlab-ci.yml@refs/tags/v1.2.3 \
+  --certificate-oidc-issuer https://gitlab.com
+------------------------------------------------------------
+```
+
+This can be used to verify the container, without the need for keys.
+
+To read more about keyless verification, using OpenID Connect (OIDC),
+please review [the Sigstore docs](https://docs.sigstore.dev/verifying/verify/#keyless-verification-using-openid-connect).
+
+### Caching
+
+By default, caching is enabled for Docker tasks.

docker.yml

+# Note: this is not a full template, just the basis
+
+include:
+  - local: .gitlab-ci-other-versions.yml
+
+.docker_buildx_base:
+  image:
+    name: registry.gitlab.com/gitlab-com/gl-infra/common-ci-tasks-images/docker:latest
+    entrypoint: [""]
+  retry: 2
+  services:
+    - docker:dind
+  variables:
+    DOCKER_DRIVER: overlayfs
+    DOCKER_HOST: tcp://docker:2376
+    DOCKER_TLS_VERIFY: 1
+    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_CERT_PATH: "/certs/client"
+    TEST_INTEGRATION_USE_SNAPSHOTTER: "true"
+
+    DOCKER_BUILD_FILE: Dockerfile # the Dockerfile to build
+    DOCKER_BUILD_CONTEXT: .  # the build context to use
+    DOCKER_BUILDX_EXTRA_ARGS: "" # additional arguments for docker buildx build
+
+    GL_VERSION_YML_FILES: .gitlab-ci-asdf-versions.yml .gitlab-ci-other-versions.yml
+    DOCKER_DESTINATION: ""  # This needs to be filled in
+
+    COSIGN_YES: "true"  # Used by Cosign to skip confirmation prompts for non-destructive operations
+
+  # See https://docs.gitlab.com/ee/ci/yaml/signing_examples.html for more details.
+  id_tokens:
+    SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio
+      aud: sigstore
+
+  tags:
+    - gitlab-org-docker
+
+  script:
+    - mkdir -p $CI_PROJECT_DIR/cache
+    - |
+      echo "Cache size at start:"
+      du -h -d0 "$CI_PROJECT_DIR/cache"
+    - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" --password-stdin
+
+    - echo "Running docker buildx build for ${DOCKER_DESTINATION}"
+    - |
+      generate_build_args() {
+        for i in "$@"; do
+          if [[ -f $i ]]; then
+            yq --unwrapScalar '.variables|to_entries[]|"--build-arg " + .key + "=" + .value + ""' "$i"
+          fi;
+        done
+      }
+      docker_version_build_args=$(generate_build_args $GL_VERSION_YML_FILES)
+
+      # Don't accidentally allow these files to be added to the
+      # Docker image
+      echo cache/ >> .dockerignore
+      echo .git/ >> .dockerignore
+
+      set -x
+      time docker buildx build \
+        -f "${DOCKER_BUILD_FILE}" \
+        $docker_version_build_args \
+        --cache-from "type=local,src=$CI_PROJECT_DIR/cache" \
+        --cache-to "type=local,dest=$CI_PROJECT_DIR/cache" \
+        --iidfile "$CI_PROJECT_DIR/.docker-imageid" \
+        --tag "${DOCKER_DESTINATION}" \
+        --push \
+        $DOCKER_BUILDX_EXTRA_ARGS \
+        "$DOCKER_BUILD_CONTEXT"
+      set +x
+
+      IMAGE_DIGEST=$(cat "$CI_PROJECT_DIR/.docker-imageid")
+
+      set -x
+      cosign sign "${DOCKER_DESTINATION}@$IMAGE_DIGEST"
+      set +x
+
+      if [[ -n $CI_COMMIT_TAG ]]; then
+        cert_ref=refs/tags/$CI_COMMIT_TAG
+      else
+        cert_ref=refs/heads/$CI_COMMIT_REF_NAME
+      fi
+
+      echo "------------------------------------------------------------"
+      echo Verify this container image using:
+      echo cosign verify "${DOCKER_DESTINATION}" \\
+      echo   --certificate-identity "${CI_PROJECT_URL}//${CI_CONFIG_PATH}@${cert_ref}" \\
+      echo   --certificate-oidc-issuer "https://$CI_SERVER_HOST"
+      echo "------------------------------------------------------------"
+
+    - |
+      echo "Cache size at end:"
+      du -h -d0 "$CI_PROJECT_DIR/cache"
+
+  after_script: |
+    cat <<-EOD
+    ----------------------------------------------------------
+    Need help? Documentation on the docker_buildx_base CI job can be found at:
+    https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docker.md
+    EOD
+  cache:
+    key: ${CI_JOB_NAME}-${DOCKER_DESTINATION}-$CI_COMMIT_REF_SLUG
+    fallback_keys:
+      - ${CI_JOB_NAME}-${DOCKER_DESTINATION}-$CI_DEFAULT_BRANCH
+    when: 'always'
+    paths:
+      - /cache

docs/developer-setup.md

@@ -21,12 +21,12 @@ Install the *Homebrew* package manager for Mac OS X.
 
 In order to protect sensitive values, many GitLab projects rely on 1password for storing secrets. This may be either directly, or through internal tooling such as [`pmv`](https://gitlab.com/gitlab-com/gl-infra/pmv).
 
-Install 1password CLI using the instructions at: <https://developer.1password.com/docs/cli/get-started#install>. Unfortunately version management or package management tools cannot be used due to the security requirements of the 1password client.
+Install 1password CLI using the instructions at: <https://developer.1password.com/docs/cli/get-started#install>.
 
-For macos, you can use:
+For MacOS, you can use:
 
 ```shell
-brew install --cask 1password/tap/1password-cli
+brew install 1password-cli
 ```
 
 Once 1password-cli is installed you can verify the installation using:
@@ -44,6 +44,16 @@ op vault list
 
 **Note: v2 of the 1password-cli is required.**
 
+#### Optional but recommended: Enable Biometrics
+
+Enabling Biometrics in the 1password-cli reduces the need to enter your password over and over again.
+
+1. Open and unlock the 1Password app.
+1. Navigate to **Settings > [Security](onepassword://settings/security)**.
+1. Turn on **Touch ID**.
+1. Navigate to **Settings > [Developer](onepassword://settings/developers)**.
+1. Turn on **Integrate with 1Password CLI**.
+
 ### Step 2: Setup Development Libraries
 
 The Infrastructure group uses some Python-based tools, including the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) and the [AWS CLI](https://aws.amazon.com/cli/), [`pre-commit`](https://pre-commit.com/) and [Ansible](https://www.ansible.com/).

editorconfig-check.md

@@ -13,7 +13,7 @@ include:
   # validate .editorconfig
   # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/editorconfig-check.md
   - project: "gitlab-com/gl-infra/common-ci-tasks"
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: "editorconfig-check.yml"
 ```
 

gitlab-scanners.md

@@ -11,6 +11,6 @@ stages:
 include:
   # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/gitlab-scanners.md
   - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: 'gitlab-scanners.yml'
 ```

gitleaks.md

@@ -37,6 +37,6 @@ include:
   # Ensure that all shell-scripts are formatted according to a
   # standard canonical format
   - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: gitleaks.yml
 ```

gitlint.md

@@ -14,6 +14,6 @@ include:
   # Runs gitlint on all terraform module directories
   # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/gitlint.md
   - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v2.26.0  # renovate:managed
+    ref: v2.27.0  # renovate:managed
     file: gitlint.yml
 ```