Extract EE specific files/lines for YAML files
With:
git diff upstream/master...ee/master --name-only -- '*.yml' ':!ee'
- .gitlab-ci.yml
- app/workers/all_queues.yml
- config/prometheus/additional_metrics.yml
- config/prometheus/cluster_metrics.yml
- config/pseudonymizer.yml
- config/sidekiq_queues.yml
- crowdin.yml
- lib/gitlab/import_export/import_export.yml
-
spec/lib/gitlab/import_export/all_models.yml: https://gitlab.com/gitlab-org/gitlab-ee/issues/7471 => https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/7254 & https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21544 - spec/lib/gitlab/import_export/safe_model_attributes.yml
Differences:
.gitlab-ci.yml
diff --git a/.gitlab-ci.yml b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/.gitlab-ci.yml
index 0afd59ab007..a82d4d5e283 100644
--- a/.gitlab-ci.yml
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/.gitlab-ci.yml
@@ -33,6 +33,11 @@ variables:
KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
BUILD_ASSETS_IMAGE: "false"
+ ## EE specific variables ##
+ # This hack is needed to make ES not that memory hungry
+ ES_JAVA_OPTS: "-Xms256m -Xmx256m"
+ ELASTIC_URL: "http://elastic:changeme@docker.elastic.co-elasticsearch-elasticsearch:9200"
+ EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master-ee.json
before_script:
- bundle --version
@@ -74,9 +79,35 @@ stages:
.use-mysql: &use-mysql
services:
- - mysql:5.7
+ - mysql:5.7.24
- redis:alpine
+# BEGIN EE-only service helpers
+
+.use-pg-9-6: &use-pg-9-6
+ services:
+ - postgres:9.6
+ - redis:alpine
+
+.use-pg-10-2: &use-pg-10-2
+ services:
+ - postgres:10.2
+ - redis:alpine
+
+.use-pg-with-elasticsearch: &use-pg-with-elasticsearch
+ services:
+ - postgres:9.6
+ - redis:alpine
+ - docker.elastic.co/elasticsearch/elasticsearch:5.6.12
+
+.use-mysql-with-elasticsearch: &use-mysql-with-elasticsearch
+ services:
+ - mysql:5.7.24
+ - redis:alpine
+ - docker.elastic.co/elasticsearch/elasticsearch:5.6.12
+
+# END EE-only service helpers
+
# Skip all jobs except the ones that begin with 'docs/'.
# Used for commits including ONLY documentation changes.
# https://docs.gitlab.com/ce/development/documentation/#testing
@@ -151,7 +182,7 @@ stages:
- '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}'
- '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}'
- scripts/gitaly-test-spawn
- - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml"
+ - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag ~geo"
artifacts:
expire_in: 31d
when: always
@@ -171,6 +202,48 @@ stages:
<<: *rspec-metadata
<<: *use-mysql
+.rspec-metadata-ee: &rspec-metadata-ee
+ <<: *rspec-metadata
+ stage: test
+ script:
+ - JOB_NAME=( $CI_JOB_NAME )
+ - TEST_TOOL=${JOB_NAME[0]}
+ - export KNAPSACK_TEST_FILE_PATTERN="ee/spec/**{,/*/**}/*_spec.rb" KNAPSACK_GENERATE_REPORT=true CACHE_CLASSES=true
+ - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+ - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
+ - scripts/gitaly-test-spawn
+ - knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag ~geo"
+
+.rspec-ee-pg: &rspec-ee-pg
+ <<: *rspec-metadata-ee
+ <<: *use-pg-with-elasticsearch
+
+.rspec-ee-mysql: &rspec-ee-mysql
+ <<: *rspec-metadata-ee
+ <<: *use-mysql-with-elasticsearch
+
+.rspec-geo-pg-9-6: &rspec-metadata-pg-geo-9-6
+ <<: *rspec-metadata
+ <<: *use-pg-9-6
+ stage: test
+ script:
+ - export NO_KNAPSACK=1
+ - export CACHE_CLASSES=true
+ - source scripts/prepare_postgres_fdw.sh
+ - scripts/gitaly-test-spawn
+ - bundle exec rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag geo ee/spec/
+
+.rspec-geo-pg-10-2: &rspec-metadata-pg-geo-10-2
+ <<: *rspec-metadata
+ <<: *use-pg-10-2
+ stage: test
+ script:
+ - export NO_KNAPSACK=1
+ - export CACHE_CLASSES=true
+ - source scripts/prepare_postgres_fdw.sh
+ - scripts/gitaly-test-spawn
+ - bundle exec rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag geo ee/spec/
+
.only-canonical-masters: &only-canonical-masters
only:
- master@gitlab-org/gitlab-ce
@@ -207,7 +280,7 @@ stages:
variables:
SETUP_DB: "false"
script:
- - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0
+ - git fetch https://gitlab.com/gitlab-org/gitlab-ee.git v9.3.0-ee
- git checkout -f FETCH_HEAD
- sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
- bundle update google-protobuf grpc
@@ -223,6 +296,24 @@ stages:
- date
- bundle exec rake db:migrate
+.migration-paths-upgrade-ce-to-ee: &migration-paths-upgrade-ce-to-ee
+ <<: *dedicated-no-docs-and-no-qa-pull-cache-job
+ variables:
+ SETUP_DB: "false"
+ script:
+ - ruby -r./scripts/ee_specific_check/ee_specific_check -e'EESpecificCheck.fetch_remote_ce_branch'
+ - git checkout -f FETCH_HEAD
+ - . scripts/utils.sh
+ - . scripts/prepare_build.sh
+ - date
+ - setup_db
+ - date
+ - git checkout -f $CI_COMMIT_SHA
+ - date
+ - . scripts/prepare_build.sh
+ - date
+ - bundle exec rake db:migrate
+
##
# Trigger a package build in omnibus-gitlab repository
#
@@ -338,6 +429,36 @@ retrieve-tests-metadata:
- mkdir -p rspec_flaky/
- wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
- '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
+ - wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH
+ - '[[ -f $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
+
+.ee-specific-check: &ee-specific-check
+ <<: *dedicated-runner
+ stage: test
+ dependencies: []
+ before_script: []
+ cache: {}
+ retry: 0
+ only:
+ - branches
+ - branches@gitlab-org/gitlab-ee
+ except:
+ - master
+ - tags
+ - /[\d-]+-stable(-ee)?/
+ - /^security-/
+ - /\bce\-to\-ee\b/
+ - /(^docs[\/-].*|.*-docs$)/
+
+ee-files-location-check:
+ <<: *ee-specific-check
+ script:
+ - scripts/ee-files-location-check
+
+ee-specific-lines-check:
+ <<: *ee-specific-check
+ script:
+ - scripts/ee-specific-lines-check
update-tests-metadata:
<<: *tests-metadata-state
@@ -352,9 +473,11 @@ update-tests-metadata:
script:
- retry gem install fog-aws mime-types activesupport --no-document
- scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
+ - scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg-ee_node_*.json
- scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
- FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
- '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH'
+ - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH'
- '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
- rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
- rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
@@ -525,6 +648,37 @@ rspec-mysql-quarantine:
<<: *rspec-quarantine
allow_failure: true
+# EE jobs START
+rspec-pg-ee:
+ <<: *rspec-ee-pg
+ parallel: 10
+
+rspec-mysql-ee:
+ <<: *rspec-ee-mysql
+ parallel: 10
+
+rspec-pg geo: *rspec-metadata-pg-geo-9-6
+rspec-pg-10 geo: *rspec-metadata-pg-geo-10-2
+
+
+.rspec-quarantine-ee: &rspec-quarantine-ee
+ retry: 0
+ script:
+ - export CACHE_CLASSES=true
+ - scripts/gitaly-test-spawn
+ - bin/rspec --color --format documentation --tag quarantine ee/spec/
+
+rspec-pg-quarantine-ee:
+ <<: *rspec-metadata-pg
+ <<: *rspec-quarantine-ee
+ allow_failure: true
+
+rspec-mysql-quarantine-ee:
+ <<: *rspec-metadata-mysql
+ <<: *rspec-quarantine-ee
+ allow_failure: true
+# EE jobs END
+
static-analysis:
<<: *dedicated-no-docs-no-db-pull-cache-job
dependencies:
@@ -547,6 +701,8 @@ docs lint:
<<: *except-qa
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
stage: test
+ variables:
+ SETUP_DB: "false"
cache: {}
dependencies: []
before_script: []
@@ -609,6 +765,14 @@ migration:path-mysql:
<<: *migration-paths
<<: *use-mysql
+migration:upgrade-pg-ce-to-ee:
+ <<: *migration-paths-upgrade-ce-to-ee
+ <<: *use-pg
+
+migration:upgrade-mysql-ce-to-ee:
+ <<: *migration-paths-upgrade-ce-to-ee
+ <<: *use-mysql
+
.db-rollback: &db-rollback
<<: *dedicated-no-docs-and-no-qa-pull-cache-job
script:
@@ -623,6 +787,13 @@ db:rollback-mysql:
<<: *db-rollback
<<: *use-mysql
+db:rollback-pg-geo:
+ <<: *db-rollback
+ <<: *use-pg
+ script:
+ - bundle exec rake geo:db:migrate VERSION=20170627195211
+ - bundle exec rake geo:db:migrate
+
gitlab:setup-pg:
<<: *gitlab-setup
<<: *use-pg
config/sidekiq_queues.yml
diff --git a/config/sidekiq_queues.yml b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/sidekiq_queues.yml
index 90cd787d5ac..aca26cd46f3 100644
--- a/config/sidekiq_queues.yml
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/config/sidekiq_queues.yml
@@ -87,3 +87,23 @@
- [remote_mirror_notification, 2]
- [import_issues_csv, 2]
- [chat_notification, 2]
+
+ # EE-specific queues
+ - [ldap_group_sync, 2]
+ - [create_github_webhook, 2]
+ - [geo, 1]
+ - [repository_update_mirror, 1]
+ - [new_epic, 2]
+ - [project_import_schedule, 1]
+ - [project_update_repository_storage, 1]
+ - [admin_emails, 1]
+ - [elastic_batch_project_indexer, 1]
+ - [elastic_indexer, 1]
+ - [elastic_commit_indexer, 1]
+ - [export_csv, 1]
+
+ # Deprecated queues: Remove after 10.7
+ - geo_base_scheduler
+ - geo_file_download
+ - geo_project_sync
+ - geo_repository_shard_sync
lib/gitlab/import_export/import_export.yml
diff --git a/lib/gitlab/import_export/import_export.yml b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/lib/gitlab/import_export/import_export.yml
index fa54fc17d95..79668a4e4c8 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/lib/gitlab/import_export/import_export.yml
@@ -66,6 +66,9 @@ project_tree:
- protected_branches:
- :merge_access_levels
- :push_access_levels
+ - :unprotect_access_levels
+ - protected_environments:
+ - :deploy_access_levels
- protected_tags:
- :create_access_levels
- :project_feature
@@ -112,10 +115,18 @@ excluded_attributes:
- :last_activity_at
- :last_repository_updated_at
- :last_repository_check_at
+ - :mirror_user_id
+ - :mirror_trigger_builds
- :storage_version
- :remote_mirror_available_overridden
+ - :only_mirror_protected_branches
+ - :pull_mirror_available_overridden
+ - :mirror_overwrites_diverged_branches
- :description_html
- :repository_languages
+ - :packages_enabled
+ - :mirror_last_update_at
+ - :mirror_last_successful_update_at
- :bfg_object_map
namespaces:
- :runners_token
@@ -123,6 +134,8 @@ excluded_attributes:
project_import_state:
- :last_error
- :jid
+ - :last_update_at
+ - :last_successful_update_at
prometheus_metrics:
- :common
- :identifier
spec/lib/gitlab/import_export/safe_model_attributes.yml
diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/spec/lib/gitlab/import_export/safe_model_attributes.yml
index baca8f6d542..cf1cff00833 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/home/yorickpeterse/Projects/gitlab/gdk-ee/gitlab/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -23,6 +23,7 @@ Issue:
- weight
- time_estimate
- relative_position
+- service_desk_reply_to
- last_edited_at
- last_edited_by_id
- discussion_locked
@@ -143,6 +144,8 @@ ProjectMember:
- invite_accepted_at
- requested_at
- expires_at
+- ldap
+- override
User:
- id
- username
@@ -486,12 +489,25 @@ Project:
- printing_merge_request_link_enabled
- resolve_outdated_diff_discussions
- build_allow_git_fetch
+- merge_requests_template
+- merge_requests_rebase_enabled
+- approvals_before_merge
+- merge_requests_author_approval
+- reset_approvals_on_push
+- disable_overriding_approvers_per_merge_request
+- merge_requests_ff_only_enabled
+- issues_template
+- repository_size_limit
+- sync_time
+- service_desk_enabled
- last_repository_updated_at
- ci_config_path
- delete_error
- merge_requests_ff_only_enabled
- merge_requests_rebase_enabled
- jobs_cache_index
+- external_authorization_classification_label
+- external_webhook_token
- pages_https_only
Author:
- name
@@ -513,12 +529,24 @@ ProtectedBranch::MergeAccessLevel:
- access_level
- created_at
- updated_at
+- user_id
+- group_id
ProtectedBranch::PushAccessLevel:
- id
- protected_branch_id
- access_level
- created_at
- updated_at
+- user_id
+- group_id
+ProtectedBranch::UnprotectAccessLevel:
+- id
+- protected_branch_id
+- access_level
+- created_at
+- updated_at
+- user_id
+- group_id
ProtectedTag::CreateAccessLevel:
- id
- protected_tag_id
@@ -581,6 +609,12 @@ PrometheusMetric:
- group
- common
- identifier
+PrometheusAlert:
+- threshold
+- operator
+- environment_id
+- project_id
+- prometheus_metric_id
Badge:
- id
- link_url
@@ -592,6 +626,20 @@ Badge:
- type
ProjectCiCdSetting:
- group_runners_enabled
+ProtectedEnvironment:
+- id
+- project_id
+- name
+- created_at
+- updated_at
+ProtectedEnvironment::DeployAccessLevel:
+- id
+- protected_environment_id
+- access_level
+- created_at
+- updated_at
+- user_id
+- group_id
ResourceLabelEvent:
- id
- action
Edited by Dave Smith