From 0cb0f3c12597ba38f08d4cf9e38e3668d28ea544 Mon Sep 17 00:00:00 2001 From: Thong Kuah Date: Thu, 27 Sep 2018 21:36:55 +1200 Subject: [PATCH 1/5] Process $DB_INITIALIZE and $DB_MIGRATE variables if $DB_INITIALIZE is present, deploy an initial release where only $DB_INITIALIZE is run in a special job (and deployments are not rendered/loaded). This is then followed by second release with deployments as usual. if $DB_MIGRATE, set this value which will trigger a pre-upgrade helm hook. --- .../48004-db-initialize-migrate.yml | 5 ++ .../ci/templates/Auto-DevOps.gitlab-ci.yml | 73 ++++++++++++++----- 2 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 changelogs/unreleased/48004-db-initialize-migrate.yml diff --git a/changelogs/unreleased/48004-db-initialize-migrate.yml b/changelogs/unreleased/48004-db-initialize-migrate.yml new file mode 100644 index 00000000000..0d691babeca --- /dev/null +++ b/changelogs/unreleased/48004-db-initialize-migrate.yml @@ -0,0 +1,5 @@ +--- +title: Support db migration and initialization for Auto DevOps +merge_request: 21955 +author: +type: added diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index e3a2534e97a..727405c7acf 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -587,26 +587,59 @@ rollout 100%: secret_name='' fi - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$CI_APPLICATION_REPOSITORY" \ - --set image.tag="$CI_APPLICATION_TAG" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.initializeCommand="$DB_INITIALIZE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + + helm upgrade --reuse-values \ + --wait \ + --set application.initializeCommand="" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + else + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" } -- GitLab From 681b7d7b49313f4bf380bace7042e7972d32b4a6 Mon Sep 17 00:00:00 2001 From: Thong Kuah Date: Tue, 2 Oct 2018 22:06:01 +1300 Subject: [PATCH 2/5] Document how to use DB_INITIALIZE and DB_MIGRATE * Limitation of DB_INITIALIZE (run once) * helm hooks * Examples --- doc/topics/autodevops/index.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 681dc8ff20d..646f716c0d0 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -440,6 +440,28 @@ no longer be valid as soon as the deployment job finishes. This means that Kubernetes can run the application, but in case it should be restarted or executed somewhere else, it cannot be accessed again. +> [Introduced][ce-21955] in GitLab 11.4 + +Database initialization and migrations can be configured to run within +the application pod by setting the project variables `DB_INITIALIZE` and +`DB_MIGRATE` respectively. + +If present, `DB_INITIALIZE` will be run as a shell command within an application pod as a helm +post-install hook. Note that this means that if any deploy succeeds, +`DB_INITIALIZE` will not be processed thereafter. + +If present, `DB_MIGRATE` will be run as a shell command within an application pod as +a helm pre-upgrade hook. + +For example, in a rails application : + +* `DB_INITIALIZE` can be set to `cd /app && RAILS_ENV=production + bin/setup` +* `DB_MIGRATE` can be set to `cd /app && RAILS_ENV=production bin/update` + +Note that `/app` is the location of the application as [configured by +Herokuish](https://github.com/gliderlabs/herokuish#paths) + > [Introduced][ce-19507] in GitLab 11.0. For internal and private projects a [GitLab Deploy Token](../../user/project/deploy_tokens/index.md###gitlab-deploy-token) @@ -581,6 +603,8 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac | `BUILDPACK_URL` | The buildpack's full URL. It can point to either Git repositories or a tarball URL. For Git repositories, it is possible to point to a specific `ref`, for example `https://github.com/heroku/heroku-buildpack-ruby.git#v142` | | `SAST_CONFIDENCE_LEVEL` | The minimum confidence level of security issues you want to be reported; `1` for Low, `2` for Medium, `3` for High; defaults to `3`.| | `DEP_SCAN_DISABLE_REMOTE_CHECKS` | Whether remote Dependency Scanning checks are disabled; defaults to `"false"`. Set to `"true"` to disable checks that send data to GitLab central servers. [Read more about remote checks](https://gitlab.com/gitlab-org/security-products/dependency-scanning#remote-checks).| +| `DB_INITIALIZE` | From GitLab 11.4, this variable can be used to specify the command to run to initialize the application's database. Run inside the application pod. | +| `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's database. Run inside the application pod. | | `STAGING_ENABLED` | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | | `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | | `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. | @@ -834,4 +858,5 @@ curl --data "value=true" --header "PRIVATE-TOKEN: personal_access_token" https:/ [postgresql]: https://www.postgresql.org/ [Auto DevOps template]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml [ee]: https://about.gitlab.com/pricing/ +[ce-21955]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21955 [ce-19507]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/19507 -- GitLab From 23ba3b02b1988205043253aee9ba91ed802f01c5 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Tue, 2 Oct 2018 15:40:56 +0300 Subject: [PATCH 3/5] Minor doc update for auto devops db migrations --- doc/topics/autodevops/index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 646f716c0d0..f67d35dac9e 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -459,7 +459,9 @@ For example, in a rails application : bin/setup` * `DB_MIGRATE` can be set to `cd /app && RAILS_ENV=production bin/update` -Note that `/app` is the location of the application as [configured by +NOTE: **Note:** +The `/app` path is the directory of your project inside the docker image +as [configured by Herokuish](https://github.com/gliderlabs/herokuish#paths) > [Introduced][ce-19507] in GitLab 11.0. -- GitLab From 597be76c1ccf31576c1c5ddd4f9d399184d0a6c1 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 3 Oct 2018 12:58:45 +0300 Subject: [PATCH 4/5] Doc improvements for DB migrations in Auto DevOps --- doc/topics/autodevops/index.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index f67d35dac9e..f4f37241659 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -442,8 +442,8 @@ executed somewhere else, it cannot be accessed again. > [Introduced][ce-21955] in GitLab 11.4 -Database initialization and migrations can be configured to run within -the application pod by setting the project variables `DB_INITIALIZE` and +Database initialization and migrations for PostgreSQL can be configured to run +within the application pod by setting the project variables `DB_INITIALIZE` and `DB_MIGRATE` respectively. If present, `DB_INITIALIZE` will be run as a shell command within an application pod as a helm @@ -453,7 +453,7 @@ post-install hook. Note that this means that if any deploy succeeds, If present, `DB_MIGRATE` will be run as a shell command within an application pod as a helm pre-upgrade hook. -For example, in a rails application : +For example, in a Rails application: * `DB_INITIALIZE` can be set to `cd /app && RAILS_ENV=production bin/setup` @@ -605,8 +605,8 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac | `BUILDPACK_URL` | The buildpack's full URL. It can point to either Git repositories or a tarball URL. For Git repositories, it is possible to point to a specific `ref`, for example `https://github.com/heroku/heroku-buildpack-ruby.git#v142` | | `SAST_CONFIDENCE_LEVEL` | The minimum confidence level of security issues you want to be reported; `1` for Low, `2` for Medium, `3` for High; defaults to `3`.| | `DEP_SCAN_DISABLE_REMOTE_CHECKS` | Whether remote Dependency Scanning checks are disabled; defaults to `"false"`. Set to `"true"` to disable checks that send data to GitLab central servers. [Read more about remote checks](https://gitlab.com/gitlab-org/security-products/dependency-scanning#remote-checks).| -| `DB_INITIALIZE` | From GitLab 11.4, this variable can be used to specify the command to run to initialize the application's database. Run inside the application pod. | -| `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's database. Run inside the application pod. | +| `DB_INITIALIZE` | From GitLab 11.4, this variable can be used to specify the command to run to initialize the application's database. It runs inside the application pod. | +| `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's database. It runs inside the application pod. | | `STAGING_ENABLED` | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | | `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | | `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. | -- GitLab From 6e564400a9533ed582f6cfc5706f321b5bdd0302 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 3 Oct 2018 13:16:38 +0300 Subject: [PATCH 5/5] Be explicit about PostgreSQL in auto devops docs --- doc/topics/autodevops/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index f4f37241659..b5a9e469965 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -605,8 +605,8 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac | `BUILDPACK_URL` | The buildpack's full URL. It can point to either Git repositories or a tarball URL. For Git repositories, it is possible to point to a specific `ref`, for example `https://github.com/heroku/heroku-buildpack-ruby.git#v142` | | `SAST_CONFIDENCE_LEVEL` | The minimum confidence level of security issues you want to be reported; `1` for Low, `2` for Medium, `3` for High; defaults to `3`.| | `DEP_SCAN_DISABLE_REMOTE_CHECKS` | Whether remote Dependency Scanning checks are disabled; defaults to `"false"`. Set to `"true"` to disable checks that send data to GitLab central servers. [Read more about remote checks](https://gitlab.com/gitlab-org/security-products/dependency-scanning#remote-checks).| -| `DB_INITIALIZE` | From GitLab 11.4, this variable can be used to specify the command to run to initialize the application's database. It runs inside the application pod. | -| `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's database. It runs inside the application pod. | +| `DB_INITIALIZE` | From GitLab 11.4, this variable can be used to specify the command to run to initialize the application's PostgreSQL database. It runs inside the application pod. | +| `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's PostgreSQL database. It runs inside the application pod. | | `STAGING_ENABLED` | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | | `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | | `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. | -- GitLab