Skip to content
Snippets Groups Projects
Verified Commit f89b08f5 authored by Timo Furrer's avatar Timo Furrer 🤹
Browse files

Remove backporting to CI/CD templates

This change set removes the machinery to backport the CI/CD component to
CI/CD templates.

Closes #43
parent 39e83294
No related branches found
No related tags found
1 merge request!71Remove backporting to CI/CD templates
......@@ -28,8 +28,6 @@ include:
- tests/integration.gitlab-ci.yml
- tests/integration-tests/*.yml
- tests/iac/**.tf
- backports/*.gitlab-ci.yml
- backports/OpenTofu/*.gitlab-ciyml
- if: $CI_COMMIT_TAG
# FIXME: we cannot make this work for all use cases because of the following:
# - cannot pass parallel.matrix to the component, thus we need to extend it
......@@ -122,7 +120,6 @@ gitlab-opentofu-image:build:
- src/**/*
- templates/**/*
- tests/**/*
- backports/**/*
check-readme:
stage: test
......@@ -142,23 +139,6 @@ check-readme:
- .gitlab/README.md.template
- templates/**/*
check-backports:
stage: test
needs: []
image: alpine:latest
before_script:
- apk add coreutils make git sed yq diffutils patch
script:
- make backports
- git diff --exit-code
rules:
- if: $CI_COMMIT_TAG
- changes:
- Makefile
- .gitlab-ci.yml
- backports/**/*
- templates/**/*
shellcheck:
stage: test
needs: []
......
......@@ -20,16 +20,6 @@ All of the above definitions have to match each other.
We currently need to change it in multiple places, because there is not a good way to share information
from the templates and the components pipeline defintion - at least in the features we'd like to use them.
## Backports
The OpenTofu CI/CD component needs to be backported as OpenTofu CI/CD template,
because components are not yet properly supported in self-managed instances.
That is, they are not bundled and it's not possible to use a component across instances.
The OpenTofu CI/CD job and pipeline templates can be generated using `make backports`.
The output is generated into the `backports` folder.
Please contribute those files only upon full manual inspection to the canonical GitLab repository.
## Releasing
Use the `make release` command with the `VERSION` argument set to the
......
all: docs backports
all: docs
.PHONY: docs
docs:
......@@ -16,41 +16,6 @@ docs:
tail -n+2 readme1 >> README.md
rm -f readme0 readme1 readme_inputs.md
BACKPORTS_DIR := backports
BACKPORTS_BASE_DIR := $(BACKPORTS_DIR)/OpenTofu
BACKPORTS_BASE_FILE := $(BACKPORTS_BASE_DIR)/Base.latest.gitlab-ci.yml
.PHONY: backports
# NOTE: this make target requires GNU sed and not the mac OS sed.
# Install it with `brew install gnu-sed` and follow the instructions in `brew info gnu-sed` to
# make it the standard `sed` binary (if you wish) or temporarily alias sed=gsed
backports:
@echo "Generating $(BACKPORTS_BASE_FILE) ..."
@mkdir -p $(BACKPORTS_BASE_DIR)
@cp $(BACKPORTS_DIR)/.Base.latest.gitlab-ci.yml $(BACKPORTS_BASE_FILE)
@sed '1,/^---$$/d' templates/fmt.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:fmt/' | sed -e 's/$$\[\[ inputs.stage \]\]/validate/' | sed -e 's/$$\[\[ inputs.allow_failure \]\]/true/' >> $(BACKPORTS_BASE_FILE)
@sed '1,/^---$$/d' templates/validate.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:validate/' | sed -e 's/$$\[\[ inputs.stage \]\]/validate/' >> $(BACKPORTS_BASE_FILE)
@sed '1,/^---$$/d' templates/plan.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:plan/' | sed -e 's/$$\[\[ inputs.stage \]\]/build/' >> $(BACKPORTS_BASE_FILE)
@sed '1,/^---$$/d' templates/apply.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:apply/' | sed -e 's/$$\[\[ inputs.stage \]\]/deploy/' | sed -e 's/"$$\[\[ inputs.auto_apply \]\]"/$$_TF_AUTO_APPLY/' >> $(BACKPORTS_BASE_FILE)
@sed '1,/^---$$/d' templates/destroy.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:destroy/' | sed -e 's/$$\[\[ inputs.stage \]\]/cleanup/' | sed -e 's/"$$\[\[ inputs.auto_destroy \]\]"/$$_TF_AUTO_DESTROY/' | sed -e 's/$$\[\[ inputs.create_destroy_job \]\]/$$TF_CREATE_DESTROY_JOB/' >> $(BACKPORTS_BASE_FILE)
@sed '1,/^---$$/d' templates/delete-state.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:delete-state/' | sed -e 's/$$\[\[ inputs.stage \]\]/cleanup/' | sed -e 's/$$\[\[ inputs.create_delete_state_job \]\]/$$TF_CREATE_DELETE_STATE_JOB/' >> $(BACKPORTS_BASE_FILE)
@# Common inputs
@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.image_registry_base \]\]/$$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/'
@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.image_name \]\]/gitlab-opentofu/'
@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.version \]\]/$$GITLAB_OPENTOFU_VERSION/'
@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.opentofu_version \]\]/$$OPENTOFU_VERSION/'
@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.root_dir \]\]/$$TF_ROOT/'
@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.state_name \]\]/$$TF_STATE_NAME/'
@sed -i $(BACKPORTS_BASE_FILE) -e '/TF_STATE_NAME: $$TF_STATE_NAME/d'
@sed -i $(BACKPORTS_BASE_FILE) -e '/TF_ROOT: $$TF_ROOT/d'
@# Remove empty variable blocks - yq doesn't preserve white spaces, thus the patching ...
@yq e 'del(.*.variables | select(length==0))' $(BACKPORTS_BASE_FILE) | diff -Bw $(BACKPORTS_BASE_FILE) - | patch $(BACKPORTS_BASE_FILE) -
@echo "Generated $(BACKPORTS_BASE_FILE)"
.PHONY: release
release:
@[ -n "$(VERSION)" ] || (echo "Please provide a VERSION argument for this release" && false)
......
# This template is a port of the OpenTofu CI/CD component at
# https://gitlab.com/components/opentofu
# It is generated with the `make backports` command from that project.
#
# Please make sure to use the component (https://gitlab.com/components/opentofu)
# when your project is hosted on GitLab.com
# or when you are willing to mirror the component project into your self-managed
# instance and use it from there.
#
# Attention: This template will be removed in favor of the OpenTofu CI/CD component following components
# are available for self-managed instances.
#
# This specific template is located at:
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenTofu/Base.latest.gitlab-ci.yml
variables:
# OpenTofu CI/CD component version, see https://gitlab.com/components/opentofu/-/releases
GITLAB_OPENTOFU_VERSION: "latest"
# Compatible OpenTofu version, see https://gitlab.com/components/opentofu/-/releases
OPENTOFU_VERSION: "1.6.0"
# Job Image with `gitlab-tofu`
GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE: $CI_REGISTRY/components/opentofu
# The relative path to the root directory of the OpenTofu project
TF_ROOT: ${CI_PROJECT_DIR}
# The name of the state file used by the GitLab Managed Terraform state backend
TF_STATE_NAME: default
opentofu:use-component-instead-of-template:
stage: .pre
needs: []
allow_failure: true
rules:
- if: '$CI_SERVER_HOST == "gitlab.com"'
image: alpine:3.19
script:
- |
echo "You are using the OpenTofu CI/CD template on GitLab.com, which is not recommended."
echo "This template is available for self-managed users only until CI/CD components are "
echo "available. See https://gitlab.com/gitlab-org/gitlab/-/issues/415638"
echo " "
echo "You should use the OpenTofu CI/CD component instead."
echo "To include the CI/CD component with a default configuration:"
echo " "
echo "include:"
echo " - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/full-pipeline@<VERSION>"
echo " inputs:"
echo " version: <VERSION>"
echo " opentofu_version: 1.6.0"
echo ""
echo "stages: [validate, build, deploy, cleanup]"
echo " "
echo "You can read about more about the OpenTofu CI/CD component here:"
echo "https://gitlab.com/components/opentofu"
- 'false'
# This template is a port of the OpenTofu CI/CD component at
# https://gitlab.com/components/opentofu
# It is generated with the `make backports` command from that project.
#
# Please make sure to use the component when your project is hosted on GitLab.com
# or when you are willing to mirror the component project into your self-managed
# instance and use it from there.
#
# Attention: This template will be removed in favor of the OpenTofu CI/CD component as soon as components
# are available for self-managed instances.
#
# This specific template is located at:
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenTofu.latest.gitlab-ci.yml
include:
- template: OpenTofu/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/OpenTofu/Base.latest.gitlab-ci.yml
stages: [validate, build, deploy]
fmt:
extends: .opentofu:fmt
validate:
extends: .opentofu:validate
plan:
extends: .opentofu:plan
apply:
extends: .opentofu:apply
# This template is a port of the OpenTofu CI/CD component at
# https://gitlab.com/components/opentofu
# It is generated with the `make backports` command from that project.
#
# Please make sure to use the component (https://gitlab.com/components/opentofu)
# when your project is hosted on GitLab.com
# or when you are willing to mirror the component project into your self-managed
# instance and use it from there.
#
# Attention: This template will be removed in favor of the OpenTofu CI/CD component following components
# are available for self-managed instances.
#
# This specific template is located at:
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenTofu/Base.latest.gitlab-ci.yml
variables:
# OpenTofu CI/CD component version, see https://gitlab.com/components/opentofu/-/releases
GITLAB_OPENTOFU_VERSION: "latest"
# Compatible OpenTofu version, see https://gitlab.com/components/opentofu/-/releases
OPENTOFU_VERSION: "1.6.0"
# Job Image with `gitlab-tofu`
GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE: $CI_REGISTRY/components/opentofu
# The relative path to the root directory of the OpenTofu project
TF_ROOT: ${CI_PROJECT_DIR}
# The name of the state file used by the GitLab Managed Terraform state backend
TF_STATE_NAME: default
opentofu:use-component-instead-of-template:
stage: .pre
needs: []
allow_failure: true
rules:
- if: '$CI_SERVER_HOST == "gitlab.com"'
image: alpine:3.19
script:
- |
echo "You are using the OpenTofu CI/CD template on GitLab.com, which is not recommended."
echo "This template is available for self-managed users only until CI/CD components are "
echo "available. See https://gitlab.com/gitlab-org/gitlab/-/issues/415638"
echo " "
echo "You should use the OpenTofu CI/CD component instead."
echo "To include the CI/CD component with a default configuration:"
echo " "
echo "include:"
echo " - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/full-pipeline@<VERSION>"
echo " inputs:"
echo " version: <VERSION>"
echo " opentofu_version: 1.6.0"
echo ""
echo "stages: [validate, build, deploy, cleanup]"
echo " "
echo "You can read about more about the OpenTofu CI/CD component here:"
echo "https://gitlab.com/components/opentofu"
- 'false'
'.opentofu:fmt':
stage: validate
needs: []
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
when: never
- if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead.
allow_failure: true
cache:
key: "$__CACHE_KEY_HACK"
paths:
- $TF_ROOT/.terraform/
variables:
# FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
__CACHE_KEY_HACK: "$TF_ROOT"
image:
name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
script:
- gitlab-tofu fmt
'.opentofu:validate':
stage: validate
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
when: never
- if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead.
cache:
key: "$__CACHE_KEY_HACK"
paths:
- $TF_ROOT/.terraform/
variables:
# FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
__CACHE_KEY_HACK: "$TF_ROOT"
image:
name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
script:
- gitlab-tofu validate
'.opentofu:plan':
stage: build
environment:
name: $TF_STATE_NAME
action: prepare
resource_group: $TF_STATE_NAME
artifacts:
# Terraform's cache files can include secrets which can be accidentally exposed.
# Please exercise caution when utilizing secrets in your Terraform infrastructure and
# consider limiting access to artifacts or take other security measures to protect sensitive information.
#
# The next line, which disables public access to pipeline artifacts, is not available on GitLab.com.
# See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic
public: false
paths:
- $TF_ROOT/plan.cache
reports:
terraform: $TF_ROOT/plan.json
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
when: never
- if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead.
cache:
key: "$__CACHE_KEY_HACK"
paths:
- $TF_ROOT/.terraform/
variables:
# FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
__CACHE_KEY_HACK: "$TF_ROOT"
image:
name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
script:
- gitlab-tofu plan
- gitlab-tofu plan-json
'.opentofu:apply':
stage: deploy
environment:
name: $TF_STATE_NAME
action: start
resource_group: $TF_STATE_NAME
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $_TF_AUTO_APPLY == "true"'
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: manual
cache:
key: "$__CACHE_KEY_HACK"
paths:
- $TF_ROOT/.terraform/
variables:
# FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
__CACHE_KEY_HACK: "$TF_ROOT"
image:
name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
script:
- gitlab-tofu apply
'.opentofu:destroy':
stage: cleanup
environment:
name: $TF_STATE_NAME
action: stop
resource_group: $TF_STATE_NAME
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $_TF_AUTO_DESTROY == "true"'
- when: manual
cache:
key: "$__CACHE_KEY_HACK"
paths:
- $TF_ROOT/.terraform/
variables:
# FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
__CACHE_KEY_HACK: "$TF_ROOT"
image:
name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
script:
- gitlab-tofu destroy
'.opentofu:delete-state':
stage: cleanup
resource_group: $TF_STATE_NAME
image: curlimages/curl:latest
script:
- curl --request DELETE -u "gitlab-ci-token:$CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/terraform/state/$TF_STATE_NAME"
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
- when: manual
# OpenTofu CI/CD template backports
See [CONTRIBUTING.md](/CONTRIBUTING.md#backports)
include:
- local: /backports/OpenTofu/Base.latest.gitlab-ci.yml
stages: [validate, test, build, deploy, cleanup]
# Required to run everything immediately, instead of manually.
fmt:
extends: .opentofu:fmt
rules: [{when: always}]
validate:
extends: .opentofu:validate
rules: [{when: always}]
plan:
extends: .opentofu:plan
rules: [{when: always}]
apply:
extends: .opentofu:apply
rules: [{when: always}]
destroy:
extends: .opentofu:destroy
rules: [{when: always}]
delete-state:
extends: .opentofu:delete-state
needs: [destroy]
rules: [{when: always}]
......@@ -12,18 +12,3 @@ component:
- PIPELINE_NAME: [Defaults]
- PIPELINE_NAME: [JobTemplates]
- PIPELINE_NAME: [TestJob]
backport-templates:
stage: test-integration
variables:
GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE: $GITLAB_OPENTOFU_IMAGE_BASE
GITLAB_OPENTOFU_VERSION: $CI_COMMIT_SHA
OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
TF_STATE_NAME: ci-integration-backports-$CI_PIPELINE_IID-$CI_NODE_INDEX
TF_ROOT: tests/iac
trigger:
include: tests/integration-tests/$PIPELINE_NAME.gitlab-ci.yml
strategy: depend
parallel:
matrix:
- PIPELINE_NAME: [BackportTemplates]
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment